From 6426960af99e6c5232b79ea9c2588bc0b27265a0 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Mon, 13 Sep 2021 21:06:36 +0300 Subject: [PATCH 001/173] add new column --- docs/en/operations/system-tables/replicas.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index e2cc607f6d8..6cdd1ecdce1 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -83,6 +83,7 @@ The next 4 columns have a non-zero value only where there is an active session w - `total_replicas` (`UInt8`) - The total number of known replicas of this table. - `active_replicas` (`UInt8`) - The number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas). - `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — Map between replica name and is replica active. +- `last_queue_update_exception`- When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions. If you request all the columns, the table may work a bit slowly, since several reads from ZooKeeper are made for each row. If you do not request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. From 0f030f868fd5f1c2bba58f458da0f02b88017143 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Mon, 13 Sep 2021 21:17:31 +0300 Subject: [PATCH 002/173] add type --- docs/en/operations/system-tables/replicas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index 6cdd1ecdce1..251c7e65537 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -83,7 +83,7 @@ The next 4 columns have a non-zero value only where there is an active session w - `total_replicas` (`UInt8`) - The total number of known replicas of this table. - `active_replicas` (`UInt8`) - The number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas). - `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — Map between replica name and is replica active. -- `last_queue_update_exception`- When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions. +- `last_queue_update_exception` (`String`) - When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions. If you request all the columns, the table may work a bit slowly, since several reads from ZooKeeper are made for each row. If you do not request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. From c428f433c3a2bfab7fc80036c8e36171ce57bec0 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 16 Sep 2021 16:38:27 +0100 Subject: [PATCH 003/173] Use do-while control loops for waiting on log processing This allows to use the same functions with very short timeouts while ensuring that the actual state is checked at least once instead of timing out before even looking at at ZK at least once. --- src/Common/ZooKeeper/ZooKeeper.cpp | 8 ++++++-- src/Storages/StorageReplicatedMergeTree.cpp | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 220b52104b5..56b96a3a45c 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -730,7 +730,10 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & } }; - while (!condition || !condition()) + /// do-while control structure to allow using this function in non-blocking + /// fashion with a wait condition which returns false by the time this + /// method is called. + do { /// Use getData insteand of exists to avoid watch leak. impl->get(path, callback, watch); @@ -746,7 +749,8 @@ bool ZooKeeper::waitForDisappear(const std::string & path, const WaitCondition & if (state->event_type == Coordination::DELETED) return true; - } + } while (!condition || !condition()); + return false; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index e0265ec0b09..47d5a1bc895 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5105,8 +5105,7 @@ bool StorageReplicatedMergeTree::tryWaitForReplicaToProcessLogEntry( /// Don't recheck ZooKeeper too often constexpr auto event_wait_timeout_ms = 3000; - if (!startsWith(entry.znode_name, "log-")) - throw Exception("Logical error: unexpected name of log node: " + entry.znode_name, ErrorCodes::LOGICAL_ERROR); + LOG_DEBUG(log, "Waiting for {} to process log entry", replica); { /// Take the number from the node name `log-xxxxxxxxxx`. @@ -5117,7 +5116,7 @@ bool StorageReplicatedMergeTree::tryWaitForReplicaToProcessLogEntry( /// Let's wait until entry gets into the replica queue. bool pulled_to_queue = false; - while (!stop_waiting()) + do { zkutil::EventPtr event = std::make_shared(); @@ -5132,8 +5131,9 @@ bool StorageReplicatedMergeTree::tryWaitForReplicaToProcessLogEntry( /// So log_pointer node will exist, but we will never update it because all background threads already stopped. /// It can lead to query hung because table drop query can wait for some query (alter, optimize, etc) which called this method, /// but the query will never finish because the drop already shut down the table. - event->tryWait(event_wait_timeout_ms); - } + if (!stop_waiting()) + event->tryWait(event_wait_timeout_ms); + } while (!stop_waiting()); if (!pulled_to_queue) return false; From 0381c634d4fbdbfbb1f88bdefc5a3545eda9ca89 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 16 Sep 2021 16:47:57 +0100 Subject: [PATCH 004/173] Add support for user defined identifier on log entries Sometimes we want to push a log entry once and only once. Because it is not possible to create a sequential node in ZooKeeper and store its name to a well known location in the same transaction we'll do it in the other order. First somehow generate a unique identifier, then submit a log entry with that identifier. Later, we can search through log entries using the identifier we provided to find the node. Required for part movement between shards. --- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 10 +++ .../MergeTree/ReplicatedMergeTreeLogEntry.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 87 ++++++++++++++++++- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 18e90952721..89515b863ca 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -26,6 +26,7 @@ enum FormatVersion : UInt8 FORMAT_WITH_DEDUPLICATE = 4, FORMAT_WITH_UUID = 5, FORMAT_WITH_DEDUPLICATE_BY_COLUMNS = 6, + FORMAT_WITH_LOG_ENTRY_ID = 7, FORMAT_LAST }; @@ -43,11 +44,17 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const if (new_part_uuid != UUIDHelpers::Nil) format_version = std::max(format_version, FORMAT_WITH_UUID); + if (!log_entry_id.empty()) + format_version = std::max(format_version, FORMAT_WITH_LOG_ENTRY_ID); + out << "format version: " << format_version << "\n" << "create_time: " << LocalDateTime(create_time ? create_time : time(nullptr)) << "\n" << "source replica: " << source_replica << '\n' << "block_id: " << escape << block_id << '\n'; + if (format_version >= FORMAT_WITH_LOG_ENTRY_ID) + out << "log_entry_id: " << escape << log_entry_id << '\n'; + switch (type) { case GET_PART: @@ -192,6 +199,9 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) in >> "block_id: " >> escape >> block_id >> "\n"; } + if (format_version >= FORMAT_WITH_LOG_ENTRY_ID) + in >> "log_entry_id: " >> escape >> log_entry_id >> "\n"; + in >> type_str >> "\n"; bool trailing_newline_found = false; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 3752c9deb8f..901669bfbdf 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -77,6 +77,7 @@ struct ReplicatedMergeTreeLogEntryData String toString() const; String znode_name; + String log_entry_id; Type type = EMPTY; String source_replica; /// Empty string means that this entry was added to the queue immediately, and not copied from the log. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 47d5a1bc895..335302e5dda 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5107,6 +5107,7 @@ bool StorageReplicatedMergeTree::tryWaitForReplicaToProcessLogEntry( LOG_DEBUG(log, "Waiting for {} to process log entry", replica); + if (startsWith(entry.znode_name, "log-")) { /// Take the number from the node name `log-xxxxxxxxxx`. UInt64 log_index = parse(entry.znode_name.substr(entry.znode_name.size() - 10)); @@ -5137,9 +5138,76 @@ bool StorageReplicatedMergeTree::tryWaitForReplicaToProcessLogEntry( if (!pulled_to_queue) return false; - } - LOG_DEBUG(log, "Looking for node corresponding to {} in {} queue", log_node_name, replica); + LOG_DEBUG(log, "Looking for node corresponding to {} in {} queue", log_node_name, replica); + } + else if (!entry.log_entry_id.empty()) + { + /// First pass, check the table log. + /// If found in the log, wait for replica to fetch it to the queue. + /// If not found in the log, it is already in the queue. + LOG_DEBUG(log, "Looking for log entry with id `{}` in the log", entry.log_entry_id); + + String log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer"); + + Strings log_entries = getZooKeeper()->getChildren(fs::path(table_zookeeper_path) / "log"); + UInt64 log_index = 0; + bool found = false; + + for (const String & log_entry_name : log_entries) + { + log_index = parse(log_entry_name.substr(log_entry_name.size() - 10)); + + if (!log_pointer.empty() && log_index < parse(log_pointer)) + continue; + + String log_entry_str; + Coordination::Stat log_entry_stat; + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "log" / log_entry_name, log_entry_str, &log_entry_stat); + ReplicatedMergeTreeLogEntryData log_entry = *ReplicatedMergeTreeLogEntry::parse(log_entry_str, log_entry_stat); + if (exists && entry.log_entry_id == log_entry.log_entry_id) + { + LOG_DEBUG(log, "Found log entry with id `{}` in the log", entry.log_entry_id); + + found = true; + log_node_name = log_entry_name; + break; + } + } + + if (found) + { + LOG_DEBUG(log, "Waiting for {} to pull {} to queue", replica, log_node_name); + + /// Let's wait until entry gets into the replica queue. + bool pulled_to_queue = false; + do + { + zkutil::EventPtr event = std::make_shared(); + + log_pointer = getZooKeeper()->get(fs::path(table_zookeeper_path) / "replicas" / replica / "log_pointer", nullptr, event); + if (!log_pointer.empty() && parse(log_pointer) > log_index) + { + pulled_to_queue = true; + break; + } + + /// Wait with timeout because we can be already shut down, but not dropped. + /// So log_pointer node will exist, but we will never update it because all background threads already stopped. + /// It can lead to query hung because table drop query can wait for some query (alter, optimize, etc) which called this method, + /// but the query will never finish because the drop already shut down the table. + if (!stop_waiting()) + event->tryWait(event_wait_timeout_ms); + } while (!stop_waiting()); + + if (!pulled_to_queue) + return false; + } + } + else + { + throw Exception("Logical error: unexpected name of log node: " + entry.znode_name, ErrorCodes::LOGICAL_ERROR); + } /** Second - find the corresponding entry in the queue of the specified replica. * Its number may not match the `log` node. Therefore, we search by comparing the content. @@ -5151,12 +5219,25 @@ bool StorageReplicatedMergeTree::tryWaitForReplicaToProcessLogEntry( for (const String & entry_name : queue_entries) { String queue_entry_str; - bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / entry_name, queue_entry_str); + Coordination::Stat queue_entry_stat; + bool exists = getZooKeeper()->tryGet(fs::path(table_zookeeper_path) / "replicas" / replica / "queue" / entry_name, queue_entry_str, &queue_entry_stat); if (exists && queue_entry_str == entry_str) { queue_entry_to_wait_for = entry_name; break; } + else if (!entry.log_entry_id.empty()) + { + /// Check if the id matches rather than just contents. This entry + /// might have been written by different ClickHouse versions and + /// it is hard to guarantee same text representation. + ReplicatedMergeTreeLogEntryData queue_entry = *ReplicatedMergeTreeLogEntry::parse(queue_entry_str, queue_entry_stat); + if (entry.log_entry_id == queue_entry.log_entry_id) + { + queue_entry_to_wait_for = entry_name; + break; + } + } } /// While looking for the record, it has already been executed and deleted. From 9a02061d9c77fc0b6a252abf80f7615f2b3a1a13 Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Thu, 16 Sep 2021 17:03:31 +0100 Subject: [PATCH 005/173] Rewrite part movement between shards logic and add kill support See https://github.com/ClickHouse/ClickHouse/pull/24585#issuecomment-857735081 for extra context about the current implementation. --- src/Access/AccessType.h | 2 +- .../InterpreterKillQueryQuery.cpp | 67 ++ src/Parsers/ASTKillQueryQuery.cpp | 16 +- src/Parsers/ASTKillQueryQuery.h | 1 + src/Parsers/ParserKillQueryQuery.cpp | 3 + src/Storages/IStorage.h | 6 + .../PartMovesBetweenShardsOrchestrator.cpp | 697 +++++++++++++----- .../PartMovesBetweenShardsOrchestrator.h | 53 +- src/Storages/StorageReplicatedMergeTree.cpp | 12 +- src/Storages/StorageReplicatedMergeTree.h | 3 + .../StorageSystemPartMovesBetweenShards.cpp | 6 +- .../test_part_moves_between_shards/test.py | 409 +++++++++- 12 files changed, 1014 insertions(+), 261 deletions(-) diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index 8e6a8be8e36..120a97d47b7 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -106,7 +106,7 @@ enum class AccessType (anyone can kill his own queries) */\ \ M(MOVE_PARTITION_BETWEEN_SHARDS, "", GLOBAL, ALL) /* required to be able to move a part/partition to a table - identified by it's ZooKeeper path */\ + identified by its ZooKeeper path */\ \ M(CREATE_USER, "", GLOBAL, ACCESS_MANAGEMENT) \ M(ALTER_USER, "", GLOBAL, ACCESS_MANAGEMENT) \ diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index d43d697fcd5..bfff1f1cd7b 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -31,6 +31,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int ACCESS_DENIED; + extern const int NOT_IMPLEMENTED; } @@ -290,6 +291,72 @@ BlockIO InterpreterKillQueryQuery::execute() break; } + case ASTKillQueryQuery::Type::PartMoveToShard: + { + if (query.sync) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "SYNC modifier is not supported for this statement."); + + Block moves_block = getSelectResult( + "database, table, task_name, task_uuid, part_name, to_shard, state", + "system.part_moves_between_shards"); + + if (!moves_block) + return res_io; + + const ColumnString & database_col = typeid_cast(*moves_block.getByName("database").column); + const ColumnString & table_col = typeid_cast(*moves_block.getByName("table").column); + const ColumnUUID & task_uuid_col = typeid_cast(*moves_block.getByName("task_uuid").column); + + auto header = moves_block.cloneEmpty(); + header.insert(0, {ColumnString::create(), std::make_shared(), "kill_status"}); + + MutableColumns res_columns = header.cloneEmptyColumns(); + auto table_id = StorageID::createEmpty(); + AccessRightsElements required_access_rights; + auto access = getContext()->getAccess(); + bool access_denied = false; + + for (size_t i = 0; i < moves_block.rows(); ++i) + { + table_id = StorageID{database_col.getDataAt(i).toString(), table_col.getDataAt(i).toString()}; + auto task_uuid = get(task_uuid_col[i]); + + CancellationCode code = CancellationCode::Unknown; + + if (!query.test) + { + auto storage = DatabaseCatalog::instance().tryGetTable(table_id, getContext()); + if (!storage) + code = CancellationCode::NotFound; + else + { + ASTAlterCommand alter_command{}; + alter_command.type = ASTAlterCommand::MOVE_PARTITION; + alter_command.move_destination_type = DataDestinationType::SHARD; + required_access_rights = InterpreterAlterQuery::getRequiredAccessForCommand( + alter_command, table_id.database_name, table_id.table_name); + if (!access->isGranted(required_access_rights)) + { + access_denied = true; + continue; + } + code = storage->killPartMoveToShard(task_uuid); + } + } + + insertResultRow(i, code, moves_block, header, res_columns); + } + + if (res_columns[0]->empty() && access_denied) + throw Exception( + "Not allowed to kill move partition. To execute this query it's necessary to have the grant " + required_access_rights.toString(), + ErrorCodes::ACCESS_DENIED); + + + res_io.in = std::make_shared(header.cloneWithColumns(std::move(res_columns))); + + break; + } } return res_io; diff --git a/src/Parsers/ASTKillQueryQuery.cpp b/src/Parsers/ASTKillQueryQuery.cpp index 72bdd7d6b0b..71c3011dd2c 100644 --- a/src/Parsers/ASTKillQueryQuery.cpp +++ b/src/Parsers/ASTKillQueryQuery.cpp @@ -11,8 +11,20 @@ String ASTKillQueryQuery::getID(char delim) const void ASTKillQueryQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { - settings.ostr << (settings.hilite ? hilite_keyword : "") << "KILL " - << (type == Type::Query ? "QUERY" : "MUTATION"); + settings.ostr << (settings.hilite ? hilite_keyword : "") << "KILL "; + + switch (type) + { + case Type::Query: + settings.ostr << "QUERY"; + break; + case Type::Mutation: + settings.ostr << "MUTATION"; + break; + case Type::PartMoveToShard: + settings.ostr << "PART_MOVE_TO_SHARD"; + break; + } formatOnCluster(settings); diff --git a/src/Parsers/ASTKillQueryQuery.h b/src/Parsers/ASTKillQueryQuery.h index c1b3956962f..6ff12bcba93 100644 --- a/src/Parsers/ASTKillQueryQuery.h +++ b/src/Parsers/ASTKillQueryQuery.h @@ -13,6 +13,7 @@ public: { Query, /// KILL QUERY Mutation, /// KILL MUTATION + PartMoveToShard, /// KILL PART_MOVE_TO_SHARD }; Type type = Type::Query; diff --git a/src/Parsers/ParserKillQueryQuery.cpp b/src/Parsers/ParserKillQueryQuery.cpp index a195a778ed2..bc895406c9f 100644 --- a/src/Parsers/ParserKillQueryQuery.cpp +++ b/src/Parsers/ParserKillQueryQuery.cpp @@ -17,6 +17,7 @@ bool ParserKillQueryQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expect ParserKeyword p_kill{"KILL"}; ParserKeyword p_query{"QUERY"}; ParserKeyword p_mutation{"MUTATION"}; + ParserKeyword p_part_move_to_shard{"PART_MOVE_TO_SHARD"}; ParserKeyword p_on{"ON"}; ParserKeyword p_test{"TEST"}; ParserKeyword p_sync{"SYNC"}; @@ -31,6 +32,8 @@ bool ParserKillQueryQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expect query->type = ASTKillQueryQuery::Type::Query; else if (p_mutation.ignore(pos, expected)) query->type = ASTKillQueryQuery::Type::Mutation; + else if (p_part_move_to_shard.ignore(pos, expected)) + query->type = ASTKillQueryQuery::Type::PartMoveToShard; else return false; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 90cb963e064..9c340100469 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -460,6 +460,12 @@ public: throw Exception("Mutations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } + /// Cancel a part move to shard. + virtual CancellationCode killPartMoveToShard(const UUID & /*task_uuid*/) + { + throw Exception("Part moves between shards are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + /** If the table have to do some complicated work on startup, * that must be postponed after creation of table object * (like launching some background threads), diff --git a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp index c227febbbc2..b3a17250549 100644 --- a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp +++ b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.cpp @@ -1,13 +1,20 @@ #include #include #include -#include #include #include #include namespace DB { + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; + extern const int TIMEOUT_EXCEEDED; +} + PartMovesBetweenShardsOrchestrator::PartMovesBetweenShardsOrchestrator(StorageReplicatedMergeTree & storage_) : storage(storage_) , zookeeper_path(storage.zookeeper_path) @@ -27,16 +34,17 @@ void PartMovesBetweenShardsOrchestrator::run() if (need_stop) return; - auto sleep_ms = 10; + /// Don't poll ZooKeeper too often. + auto sleep_ms = 3 * 1000; try { - fetchStateFromZK(); + syncStateFromZK(); + /// Schedule for immediate re-execution as likely there is more work + /// to be done. if (step()) - fetchStateFromZK(); - else - sleep_ms = 3 * 1000; + task->schedule(); } catch (...) { @@ -54,11 +62,11 @@ void PartMovesBetweenShardsOrchestrator::shutdown() LOG_TRACE(log, "PartMovesBetweenShardsOrchestrator thread finished"); } -void PartMovesBetweenShardsOrchestrator::fetchStateFromZK() +void PartMovesBetweenShardsOrchestrator::syncStateFromZK() { std::lock_guard lock(state_mutex); - entries.clear(); + std::vector new_entries; auto zk = storage.getZooKeeper(); @@ -76,8 +84,11 @@ void PartMovesBetweenShardsOrchestrator::fetchStateFromZK() e.version = stat.version; e.znode_name = task_name; - entries[task_name] = std::move(e); + new_entries.push_back(std::move(e)); } + + // Replace in-memory state. + entries = new_entries; } bool PartMovesBetweenShardsOrchestrator::step() @@ -93,7 +104,7 @@ bool PartMovesBetweenShardsOrchestrator::step() { std::lock_guard lock(state_mutex); - for (auto const & entry : entries | boost::adaptors::map_values) + for (auto const & entry : entries) { if (entry.state.value == EntryState::DONE || entry.state.value == EntryState::CANCELLED) continue; @@ -128,11 +139,21 @@ bool PartMovesBetweenShardsOrchestrator::step() throw; } + LOG_DEBUG(log, "stepEntry on task {} from state {} (rollback: {}), try: {}", + entry_to_process->znode_name, + entry_to_process->state.toString(), + entry_to_process->rollback, + entry_to_process->num_tries); + try { /// Use the same ZooKeeper connection. If we'd lost the lock then connection /// will become expired and all consequent operations will fail. - stepEntry(entry_to_process.value(), zk); + Entry new_entry = stepEntry(entry_to_process.value(), zk); + new_entry.last_exception_msg = ""; + new_entry.num_tries = 0; + new_entry.update_time = std::time(nullptr); + zk->set(new_entry.znode_path, new_entry.toString(), new_entry.version); } catch (...) { @@ -140,6 +161,7 @@ bool PartMovesBetweenShardsOrchestrator::step() Entry entry_copy = entry_to_process.value(); entry_copy.last_exception_msg = getCurrentExceptionMessage(false); + entry_copy.num_tries += 1; entry_copy.update_time = std::time(nullptr); zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); @@ -149,276 +171,537 @@ bool PartMovesBetweenShardsOrchestrator::step() return true; } -void PartMovesBetweenShardsOrchestrator::stepEntry(const Entry & entry, zkutil::ZooKeeperPtr zk) +PartMovesBetweenShardsOrchestrator::Entry PartMovesBetweenShardsOrchestrator::stepEntry(Entry entry, zkutil::ZooKeeperPtr zk) { switch (entry.state.value) { - case EntryState::DONE: - break; - + case EntryState::DONE: [[fallthrough]]; case EntryState::CANCELLED: - break; + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't stepEntry after terminal state. This is a bug."); case EntryState::TODO: { - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::SYNC_SOURCE; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + if (entry.rollback) + { + removePins(entry, zk); + entry.state = EntryState::CANCELLED; + return entry; + } + /// The forward transition happens implicitly when task is created by `StorageReplicatedMergeTree::movePartitionToShard`. + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected entry state ({}) in stepEntry. This is a bug.", entry.state.toString()); } - break; case EntryState::SYNC_SOURCE: { + if (entry.rollback) { - /// Log entry. - Coordination::Requests ops; - ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); - - ReplicatedMergeTreeLogEntryData log_entry; - log_entry.type = ReplicatedMergeTreeLogEntryData::SYNC_PINNED_PART_UUIDS; - log_entry.create_time = std::time(nullptr); - log_entry.source_replica = storage.replica_name; - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/log/log-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); - - Coordination::Responses responses; - Coordination::Error rc = zk->tryMulti(ops, responses); - zkutil::KeeperMultiException::check(rc, ops, responses); - - String log_znode_path = dynamic_cast(*responses.back()).path_created; - log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); - - /// This wait in background schedule pool is useless. It'd be - /// better to have some notification which will call `step` - /// function when all replicated will finish. TODO. - storage.waitForAllReplicasToProcessLogEntry(zookeeper_path, log_entry, -1); + entry.state = EntryState::TODO; + return entry; } - + else { - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::SYNC_DESTINATION; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + ReplicatedMergeTreeLogEntryData sync_source_log_entry; + + String sync_source_log_entry_barrier_path = fs::path(entry.znode_path) / ("log_" + entry.state.toString()); + Coordination::Stat sync_source_log_entry_stat; + String sync_source_log_entry_str; + if (zk->tryGet(sync_source_log_entry_barrier_path, sync_source_log_entry_str, &sync_source_log_entry_stat)) + { + LOG_DEBUG(log, "Log entry was already created will check the existing one."); + + sync_source_log_entry = *ReplicatedMergeTreeLogEntry::parse(sync_source_log_entry_str, sync_source_log_entry_stat); + } + else + { + /// Log entry. + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); + + sync_source_log_entry.type = ReplicatedMergeTreeLogEntryData::SYNC_PINNED_PART_UUIDS; + sync_source_log_entry.log_entry_id = sync_source_log_entry_barrier_path; + sync_source_log_entry.create_time = std::time(nullptr); + sync_source_log_entry.source_replica = storage.replica_name; + + ops.emplace_back(zkutil::makeCreateRequest(sync_source_log_entry_barrier_path, sync_source_log_entry.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/log/log-", sync_source_log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + + Coordination::Responses responses; + Coordination::Error rc = zk->tryMulti(ops, responses); + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path = dynamic_cast(*responses.back()).path_created; + sync_source_log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + LOG_DEBUG(log, "Pushed log entry: {}", log_znode_path); + } + + Strings unwaited = storage.tryWaitForAllReplicasToProcessLogEntry(zookeeper_path, sync_source_log_entry, 1); + if (!unwaited.empty()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Some replicas haven't processed event: {}, will retry later.", toString(unwaited)); + + entry.state = EntryState::SYNC_DESTINATION; + return entry; } } - break; case EntryState::SYNC_DESTINATION: { + if (entry.rollback) { - /// Log entry. - Coordination::Requests ops; - ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); - - ReplicatedMergeTreeLogEntryData log_entry; - log_entry.type = ReplicatedMergeTreeLogEntryData::SYNC_PINNED_PART_UUIDS; - log_entry.create_time = std::time(nullptr); - log_entry.source_replica = storage.replica_name; - log_entry.source_shard = zookeeper_path; - - ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest( - entry.to_shard + "/log/log-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); - - Coordination::Responses responses; - Coordination::Error rc = zk->tryMulti(ops, responses); - zkutil::KeeperMultiException::check(rc, ops, responses); - - String log_znode_path = dynamic_cast(*responses.back()).path_created; - log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); - - storage.waitForAllReplicasToProcessLogEntry(entry.to_shard, log_entry, -1); - } - - { - /// State transition. Entry entry_copy = entry; - entry_copy.state = EntryState::DESTINATION_FETCH; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + entry_copy.state = EntryState::SYNC_SOURCE; + return entry_copy; + } + else + { + ReplicatedMergeTreeLogEntryData sync_destination_log_entry; + + String sync_destination_log_entry_barrier_path = fs::path(entry.znode_path) / ("log_" + entry.state.toString()); + Coordination::Stat sync_destination_log_entry_stat; + String sync_destination_log_entry_str; + if (zk->tryGet(sync_destination_log_entry_barrier_path, sync_destination_log_entry_str, &sync_destination_log_entry_stat)) + { + LOG_DEBUG(log, "Log entry was already created will check the existing one."); + + sync_destination_log_entry = *ReplicatedMergeTreeLogEntry::parse(sync_destination_log_entry_str, sync_destination_log_entry_stat); + } + else + { + /// Log entry. + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); + + sync_destination_log_entry.type = ReplicatedMergeTreeLogEntryData::SYNC_PINNED_PART_UUIDS; + sync_destination_log_entry.log_entry_id = sync_destination_log_entry_barrier_path; + sync_destination_log_entry.create_time = std::time(nullptr); + sync_destination_log_entry.source_replica = storage.replica_name; + sync_destination_log_entry.source_shard = zookeeper_path; + + ops.emplace_back(zkutil::makeCreateRequest(sync_destination_log_entry_barrier_path, sync_destination_log_entry.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest( + entry.to_shard + "/log/log-", sync_destination_log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + + Coordination::Responses responses; + Coordination::Error rc = zk->tryMulti(ops, responses); + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path = dynamic_cast(*responses.back()).path_created; + sync_destination_log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + LOG_DEBUG(log, "Pushed log entry: {}", log_znode_path); + } + + Strings unwaited = storage.tryWaitForAllReplicasToProcessLogEntry(entry.to_shard, sync_destination_log_entry, 1); + if (!unwaited.empty()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Some replicas haven't processed event: {}, will retry later.", toString(unwaited)); + + entry.state = EntryState::DESTINATION_FETCH; + return entry; } } - break; case EntryState::DESTINATION_FETCH: { - /// Make sure table structure doesn't change when there are part movements in progress. + + if (entry.rollback) { - Coordination::Requests ops; - ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); + // TODO(nv): Do we want to cleanup fetched data on the destination? + // Maybe leave it there and make sure a background cleanup will take + // care of it sometime later. - /// Log entry. - ReplicatedMergeTreeLogEntryData log_entry; - log_entry.type = ReplicatedMergeTreeLogEntryData::CLONE_PART_FROM_SHARD; - log_entry.create_time = std::time(nullptr); - log_entry.new_part_name = entry.part_name; - log_entry.source_replica = storage.replica_name; - log_entry.source_shard = zookeeper_path; - ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest( - entry.to_shard + "/log/log-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); - - Coordination::Responses responses; - Coordination::Error rc = zk->tryMulti(ops, responses); - zkutil::KeeperMultiException::check(rc, ops, responses); - - String log_znode_path = dynamic_cast(*responses.back()).path_created; - log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); - - storage.waitForAllReplicasToProcessLogEntry(entry.to_shard, log_entry, -1); + entry.state = EntryState::SYNC_DESTINATION; + return entry; } - + else { - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::DESTINATION_ATTACH; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + /// Note: Table structure shouldn't be changed while there are part movements in progress. + + ReplicatedMergeTreeLogEntryData fetch_log_entry; + + String fetch_log_entry_barrier_path = fs::path(entry.znode_path) / ("log_" + entry.state.toString()); + Coordination::Stat fetch_log_entry_stat; + String fetch_log_entry_str; + if (zk->tryGet(fetch_log_entry_barrier_path, fetch_log_entry_str, &fetch_log_entry_stat)) + { + LOG_DEBUG(log, "Log entry was already created will check the existing one."); + + fetch_log_entry = *ReplicatedMergeTreeLogEntry::parse(fetch_log_entry_str, fetch_log_entry_stat); + } + else + { + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); + + fetch_log_entry.type = ReplicatedMergeTreeLogEntryData::CLONE_PART_FROM_SHARD; + fetch_log_entry.log_entry_id = fetch_log_entry_barrier_path; + fetch_log_entry.create_time = std::time(nullptr); + fetch_log_entry.new_part_name = entry.part_name; + fetch_log_entry.source_replica = storage.replica_name; + fetch_log_entry.source_shard = zookeeper_path; + ops.emplace_back(zkutil::makeCreateRequest(fetch_log_entry_barrier_path, fetch_log_entry.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest( + entry.to_shard + "/log/log-", fetch_log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + + Coordination::Responses responses; + Coordination::Error rc = zk->tryMulti(ops, responses); + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path = dynamic_cast(*responses.back()).path_created; + fetch_log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + LOG_DEBUG(log, "Pushed log entry: {}", log_znode_path); + } + + Strings unwaited = storage.tryWaitForAllReplicasToProcessLogEntry(entry.to_shard, fetch_log_entry, 1); + if (!unwaited.empty()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Some replicas haven't processed event: {}, will retry later.", toString(unwaited)); + + entry.state = EntryState::DESTINATION_ATTACH; + return entry; } } - break; case EntryState::DESTINATION_ATTACH: { - /// There is a chance that attach on destination will fail and this task will be left in the queue forever. + String attach_log_entry_barrier_path = fs::path(entry.znode_path) / ("log_" + entry.state.toString()); + + if (entry.rollback) { + Coordination::Stat attach_log_entry_stat; + String attach_log_entry_str; + if (!zk->tryGet(attach_log_entry_barrier_path, attach_log_entry_str, &attach_log_entry_stat)) + { + LOG_DEBUG(log, "Log entry for DESTINATION_ATTACH not found. Not sending DROP_RANGE log entry."); + + // ATTACH_PART wasn't issued, nothing to revert. + entry.state = EntryState::DESTINATION_FETCH; + return entry; + } + else + { + // Need to remove ATTACH_PART from the queue or drop data. + // Similar to `StorageReplicatedMergeTree::dropPart` w/o extra + // checks as we know drop shall be possible. + ReplicatedMergeTreeLogEntryData attach_rollback_log_entry; + + String attach_rollback_log_entry_barrier_path = fs::path(entry.znode_path) / ("log_" + entry.state.toString() + "_rollback"); + Coordination::Stat attach_rollback_log_entry_stat; + String attach_rollback_log_entry_str; + if (zk->tryGet(attach_rollback_log_entry_barrier_path, attach_rollback_log_entry_str, &attach_rollback_log_entry_stat)) + { + LOG_DEBUG(log, "Log entry was already created will check the existing one."); + + attach_rollback_log_entry = *ReplicatedMergeTreeLogEntry::parse(attach_rollback_log_entry_str, attach_rollback_log_entry_stat); + } + else + { + const auto attach_log_entry = ReplicatedMergeTreeLogEntry::parse(attach_log_entry_str, attach_log_entry_stat); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); + + auto drop_part_info = MergeTreePartInfo::fromPartName(attach_log_entry->new_part_name, storage.format_version); + + storage.getClearBlocksInPartitionOps( + ops, *zk, drop_part_info.partition_id, drop_part_info.min_block, drop_part_info.max_block); + size_t clear_block_ops_size = ops.size(); + + attach_rollback_log_entry.type = ReplicatedMergeTreeLogEntryData::DROP_RANGE; + attach_rollback_log_entry.log_entry_id = attach_rollback_log_entry_barrier_path; + attach_rollback_log_entry.source_replica = storage.replica_name; + attach_rollback_log_entry.source_shard = zookeeper_path; + + attach_rollback_log_entry.new_part_name = getPartNamePossiblyFake(storage.format_version, drop_part_info); + attach_rollback_log_entry.create_time = time(nullptr); + + ops.emplace_back(zkutil::makeCreateRequest(attach_rollback_log_entry_barrier_path, attach_rollback_log_entry.toString(), -1)); + ops.emplace_back(zkutil::makeCreateRequest( + entry.to_shard + "/log/log-", attach_rollback_log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); + Coordination::Responses responses; + Coordination::Error rc = zk->tryMulti(ops, responses); + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path + = dynamic_cast(*responses[clear_block_ops_size]).path_created; + attach_rollback_log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + LOG_DEBUG(log, "Pushed log entry: {}", log_znode_path); + } + + Strings unwaited = storage.tryWaitForAllReplicasToProcessLogEntry(entry.to_shard, attach_rollback_log_entry, 1); + if (!unwaited.empty()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Some replicas haven't processed event: {}, will retry later.", toString(unwaited)); + + entry.state = EntryState::DESTINATION_FETCH; + return entry; + } + } + else + { + /// There is a chance that attach on destination will fail and this task will be left in the queue forever. + Coordination::Requests ops; ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); auto part = storage.getActiveContainingPart(entry.part_name); + /// Allocating block number in other replicas zookeeper path /// TODO Maybe we can do better. - auto block_number_lock = storage.allocateBlockNumber(part->info.partition_id, zk, "", entry.to_shard); - auto block_number = block_number_lock->getNumber(); + auto block_number_lock = storage.allocateBlockNumber(part->info.partition_id, zk, attach_log_entry_barrier_path, entry.to_shard); - auto part_info = part->info; - part_info.min_block = block_number; - part_info.max_block = block_number; - part_info.level = 0; - part_info.mutation = 0; - - /// Attach log entry (all replicas already fetched part) ReplicatedMergeTreeLogEntryData log_entry; - log_entry.type = ReplicatedMergeTreeLogEntryData::ATTACH_PART; - log_entry.part_checksum = part->checksums.getTotalChecksumHex(); - log_entry.create_time = std::time(nullptr); - log_entry.new_part_name = part_info.getPartName(); - ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); - ops.emplace_back(zkutil::makeCreateRequest( - entry.to_shard + "/log/log-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); - Coordination::Responses responses; - Coordination::Error rc = zk->tryMulti(ops, responses); - zkutil::KeeperMultiException::check(rc, ops, responses); + if (block_number_lock) + { + auto block_number = block_number_lock->getNumber(); - String log_znode_path = dynamic_cast(*responses.back()).path_created; - log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + auto part_info = part->info; + part_info.min_block = block_number; + part_info.max_block = block_number; + part_info.level = 0; + part_info.mutation = 0; - storage.waitForAllReplicasToProcessLogEntry(entry.to_shard, log_entry, -1); - } + /// Attach log entry (all replicas already fetched part) + log_entry.type = ReplicatedMergeTreeLogEntryData::ATTACH_PART; + log_entry.log_entry_id = attach_log_entry_barrier_path; + log_entry.part_checksum = part->checksums.getTotalChecksumHex(); + log_entry.create_time = std::time(nullptr); + log_entry.new_part_name = part_info.getPartName(); - { - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::SOURCE_DROP_PRE_DELAY; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + ops.emplace_back(zkutil::makeCreateRequest(attach_log_entry_barrier_path, log_entry.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest( + entry.to_shard + "/log/log-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + + Coordination::Responses responses; + Coordination::Error rc = zk->tryMulti(ops, responses); + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path = dynamic_cast(*responses.back()).path_created; + log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + LOG_DEBUG(log, "Pushed log entry: {}", log_znode_path); + } + else + { + LOG_DEBUG(log, "Log entry was already created will check the existing one."); + + Coordination::Stat stat; + String log_entry_str = zk->get(attach_log_entry_barrier_path, &stat); + log_entry = *ReplicatedMergeTreeLogEntry::parse(log_entry_str, stat); + } + + Strings unwaited = storage.tryWaitForAllReplicasToProcessLogEntry(entry.to_shard, log_entry, 1); + if (!unwaited.empty()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Some replicas haven't processed event: {}, will retry later.", toString(unwaited)); + + + entry.dst_part_name = log_entry.new_part_name; + entry.state = EntryState::SOURCE_DROP_PRE_DELAY; + return entry; } } - break; case EntryState::SOURCE_DROP_PRE_DELAY: { - std::this_thread::sleep_for(std::chrono::seconds(storage.getSettings()->part_moves_between_shards_delay_seconds)); - - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::SOURCE_DROP; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + if (entry.rollback) + { + entry.state = EntryState::DESTINATION_ATTACH; + return entry; + } + else + { + std::this_thread::sleep_for(std::chrono::seconds(storage.getSettings()->part_moves_between_shards_delay_seconds)); + entry.state = EntryState::SOURCE_DROP; + return entry; + } } - break; case EntryState::SOURCE_DROP: { + if (entry.rollback) + throw Exception(ErrorCodes::LOGICAL_ERROR, "It is not possible to rollback from this state. This is a bug."); + else { - ReplicatedMergeTreeLogEntry log_entry; - if (storage.dropPartImpl(zk, entry.part_name, log_entry, false, false)) - storage.waitForAllReplicasToProcessLogEntry(zookeeper_path, log_entry, -1); - } + // Can't use dropPartImpl directly as we need additional zk ops to remember the log entry + // for subsequent retries. - { - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::SOURCE_DROP_POST_DELAY; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + ReplicatedMergeTreeLogEntryData source_drop_log_entry; + + String source_drop_log_entry_barrier_path = fs::path(entry.znode_path) / ("log_" + entry.state.toString()); + Coordination::Stat source_drop_log_entry_stat; + String source_drop_log_entry_str; + if (zk->tryGet(source_drop_log_entry_barrier_path, source_drop_log_entry_str, &source_drop_log_entry_stat)) + { + LOG_DEBUG(log, "Log entry was already created will check the existing one."); + + source_drop_log_entry = *ReplicatedMergeTreeLogEntry::parse(source_drop_log_entry_str, source_drop_log_entry_stat); + } + else + { + auto source_drop_part_info = MergeTreePartInfo::fromPartName(entry.part_name, storage.format_version); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(entry.znode_path, entry.version)); + + storage.getClearBlocksInPartitionOps(ops, *zk, source_drop_part_info.partition_id, source_drop_part_info.min_block, source_drop_part_info.max_block); + + source_drop_log_entry.type = ReplicatedMergeTreeLogEntryData::DROP_RANGE; + source_drop_log_entry.log_entry_id = source_drop_log_entry_barrier_path; + source_drop_log_entry.create_time = std::time(nullptr); + source_drop_log_entry.new_part_name = getPartNamePossiblyFake(storage.format_version, source_drop_part_info); + source_drop_log_entry.source_replica = storage.replica_name; + + ops.emplace_back(zkutil::makeCreateRequest(source_drop_log_entry_barrier_path, source_drop_log_entry.toString(), -1)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/log", "", -1)); + ops.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/log/log-", source_drop_log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + + Coordination::Responses responses; + Coordination::Error rc = zk->tryMulti(ops, responses); + zkutil::KeeperMultiException::check(rc, ops, responses); + + String log_znode_path = dynamic_cast(*responses.back()).path_created; + source_drop_log_entry.znode_name = log_znode_path.substr(log_znode_path.find_last_of('/') + 1); + + LOG_DEBUG(log, "Pushed log entry: {}", log_znode_path); + } + + Strings unwaited = storage.tryWaitForAllReplicasToProcessLogEntry(zookeeper_path, source_drop_log_entry, 1); + if (!unwaited.empty()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Some replicas haven't processed event: {}, will retry later.", toString(unwaited)); + + entry.state = EntryState::SOURCE_DROP_POST_DELAY; + return entry; } } - break; case EntryState::SOURCE_DROP_POST_DELAY: { - std::this_thread::sleep_for(std::chrono::seconds(storage.getSettings()->part_moves_between_shards_delay_seconds)); - - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::REMOVE_UUID_PIN; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); + if (entry.rollback) + throw Exception(ErrorCodes::LOGICAL_ERROR, "It is not possible to rollback from this state. This is a bug."); + else + { + std::this_thread::sleep_for(std::chrono::seconds(storage.getSettings()->part_moves_between_shards_delay_seconds)); + entry.state = EntryState::REMOVE_UUID_PIN; + return entry; + } } - break; case EntryState::REMOVE_UUID_PIN: { + if (entry.rollback) + throw Exception(ErrorCodes::LOGICAL_ERROR, "It is not possible to rollback from this state. This is a bug."); + else { - PinnedPartUUIDs src_pins; - PinnedPartUUIDs dst_pins; + removePins(entry, zk); - { - String s = zk->get(zookeeper_path + "/pinned_part_uuids", &src_pins.stat); - src_pins.fromString(s); - } - - { - String s = zk->get(entry.to_shard + "/pinned_part_uuids", &dst_pins.stat); - dst_pins.fromString(s); - } - - src_pins.part_uuids.erase(entry.part_uuid); - dst_pins.part_uuids.erase(entry.part_uuid); - - Coordination::Requests ops; - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/pinned_part_uuids", src_pins.toString(), src_pins.stat.version)); - ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/pinned_part_uuids", dst_pins.toString(), dst_pins.stat.version)); - - zk->multi(ops); + entry.state = EntryState::DONE; + return entry; } - - /// State transition. - Entry entry_copy = entry; - entry_copy.state = EntryState::DONE; - entry_copy.update_time = std::time(nullptr); - zk->set(entry_copy.znode_path, entry_copy.toString(), entry_copy.version); } - break; + } + + __builtin_unreachable(); +} + +void PartMovesBetweenShardsOrchestrator::removePins(const Entry & entry, zkutil::ZooKeeperPtr zk) +{ + PinnedPartUUIDs src_pins; + PinnedPartUUIDs dst_pins; + + { + String s = zk->get(zookeeper_path + "/pinned_part_uuids", &src_pins.stat); + src_pins.fromString(s); + } + + { + String s = zk->get(entry.to_shard + "/pinned_part_uuids", &dst_pins.stat); + dst_pins.fromString(s); + } + + dst_pins.part_uuids.erase(entry.part_uuid); + src_pins.part_uuids.erase(entry.part_uuid); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/pinned_part_uuids", src_pins.toString(), src_pins.stat.version)); + ops.emplace_back(zkutil::makeSetRequest(entry.to_shard + "/pinned_part_uuids", dst_pins.toString(), dst_pins.stat.version)); + + zk->multi(ops); +} + +CancellationCode PartMovesBetweenShardsOrchestrator::killPartMoveToShard(const UUID & task_uuid) +{ + while (true) + { + auto entry = getEntryByUUID(task_uuid); + + // If the task is in this state or any that follows it is too late to rollback + // since we can't be sure if the source data still exists. + auto not_possible_to_rollback_after_state = EntryState(EntryState::SOURCE_DROP); + if (entry.state.value >= not_possible_to_rollback_after_state.value) + { + LOG_DEBUG(log, "Can't kill move part between shards entry {} ({}) after state {}. Current state: {}.", + toString(entry.task_uuid), entry.znode_name, not_possible_to_rollback_after_state.toString(), entry.state.toString()); + return CancellationCode::CancelCannotBeSent; + } + + LOG_TRACE(log, "Will try to mark move part between shards entry {} ({}) for rollback.", + toString(entry.task_uuid), entry.znode_name); + + auto zk = storage.getZooKeeper(); + + // State transition. + entry.rollback = true; + entry.update_time = std::time(nullptr); + entry.num_tries = 0; + entry.last_exception_msg = ""; + + auto code = zk->trySet(entry.znode_path, entry.toString(), entry.version); + if (code == Coordination::Error::ZOK) + { + // Orchestrator will process it in background. + return CancellationCode::CancelSent; + } + else if (code == Coordination::Error::ZBADVERSION) + { + /// Node was updated meanwhile. We must re-read it and repeat all the actions. + continue; + } + else + throw Coordination::Exception(code, entry.znode_path); } } -std::vector PartMovesBetweenShardsOrchestrator::getEntries() const +std::vector PartMovesBetweenShardsOrchestrator::getEntries() { + // Force sync. Also catches parsing errors. + syncStateFromZK(); + std::lock_guard lock(state_mutex); - std::vector res; + return entries; +} - for (const auto & e : entries) - res.push_back(e.second); +PartMovesBetweenShardsOrchestrator::Entry PartMovesBetweenShardsOrchestrator::getEntryByUUID(const UUID & task_uuid) +{ + /// Need latest state in case user tries to kill a move observed on a different replica. + syncStateFromZK(); - return res; + std::lock_guard lock(state_mutex); + for (auto const & entry : entries) + { + if (entry.task_uuid == task_uuid) + return entry; + } + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Task with id {} not found", toString(task_uuid)); } String PartMovesBetweenShardsOrchestrator::Entry::toString() const @@ -431,12 +714,19 @@ String PartMovesBetweenShardsOrchestrator::Entry::toString() const json.set(JSON_KEY_PART_NAME, part_name); json.set(JSON_KEY_PART_UUID, DB::toString(part_uuid)); json.set(JSON_KEY_TO_SHARD, to_shard); + json.set(JSON_KEY_DST_PART_NAME, dst_part_name); json.set(JSON_KEY_STATE, state.toString()); + json.set(JSON_KEY_ROLLBACK, DB::toString(rollback)); json.set(JSON_KEY_LAST_EX_MSG, last_exception_msg); + json.set(JSON_KEY_NUM_TRIES, DB::toString(num_tries)); std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); - json.stringify(oss); + + // Always escape unicode to make last_exception_msg json safe. + // It may contain random binary data when exception is a parsing error + // of unexpected contents. + Poco::JSON::Stringifier::stringify(json, oss, 0, -1, Poco::JSON_WRAP_STRINGS | Poco::JSON_ESCAPE_UNICODE); return oss.str(); } @@ -452,8 +742,11 @@ void PartMovesBetweenShardsOrchestrator::Entry::fromString(const String & buf) part_name = json->getValue(JSON_KEY_PART_NAME); part_uuid = parseFromString(json->getValue(JSON_KEY_PART_UUID)); to_shard = json->getValue(JSON_KEY_TO_SHARD); + dst_part_name = json->getValue(JSON_KEY_DST_PART_NAME); state.value = EntryState::fromString(json->getValue(JSON_KEY_STATE)); + rollback = json->getValue(JSON_KEY_ROLLBACK); last_exception_msg = json->getValue(JSON_KEY_LAST_EX_MSG); + num_tries = json->getValue(JSON_KEY_NUM_TRIES); } } diff --git a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h index 9e54ae8a8ed..38b6a076748 100644 --- a/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h +++ b/src/Storages/MergeTree/PartMovesBetweenShardsOrchestrator.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { @@ -18,14 +19,37 @@ namespace ErrorCodes class StorageReplicatedMergeTree; -/// Cross shard part movement workflow orchestration. +/** + * Cross shard part movement workflow orchestration. + * + * TODO(nv): + * * Usage of `format_version` when acting on the behalf of the remote shard. + * There needs to be sort of an API to coordinate with remote replicas. + * * Only one movement at a time can be coordinated. This can easily be fixed + * by cycling through different tasks and checking their status with a + * priority queue and back-off for failing tasks + * `min(backoff * num_tries, max_backoff)`. + */ class PartMovesBetweenShardsOrchestrator { public: struct EntryState { + // State transitions are linear. When a kill query is issued a rollback + // flag is set and transitions order is reversed. + // + // SOURCE_DROP is a critical state after which rollback is not possible + // and we must ensure that the task can always succeed after that. + // + // Similar for rollback. It should be always possible to rollback before + // SOURCE_DROP state and it should terminate. + // + // Note: This fragile. If you change the states please add entry to + // changelog about forward/backward compatibility. Better not to have + // any active move tasks while doing upgrade/downgrade operations. enum Value { + CANCELLED, TODO, SYNC_SOURCE, SYNC_DESTINATION, @@ -36,7 +60,6 @@ public: SOURCE_DROP_POST_DELAY, REMOVE_UUID_PIN, DONE, - CANCELLED, }; EntryState(): value(TODO) {} @@ -95,10 +118,14 @@ public: String part_name; UUID part_uuid; String to_shard; + String dst_part_name; EntryState state; + bool rollback = false; + /// Reset on successful transitions. String last_exception_msg; + UInt64 num_tries = 0; String znode_name; @@ -120,27 +147,31 @@ private: static constexpr auto JSON_KEY_PART_NAME = "part_name"; static constexpr auto JSON_KEY_PART_UUID = "part_uuid"; static constexpr auto JSON_KEY_TO_SHARD = "to_shard"; + static constexpr auto JSON_KEY_DST_PART_NAME = "dst_part_name"; static constexpr auto JSON_KEY_STATE = "state"; + static constexpr auto JSON_KEY_ROLLBACK = "rollback"; static constexpr auto JSON_KEY_LAST_EX_MSG = "last_exception"; + static constexpr auto JSON_KEY_NUM_TRIES = "num_tries"; public: - PartMovesBetweenShardsOrchestrator(StorageReplicatedMergeTree & storage_); + explicit PartMovesBetweenShardsOrchestrator(StorageReplicatedMergeTree & storage_); void start() { task->activateAndSchedule(); } void wakeup() { task->schedule(); } void shutdown(); - void fetchStateFromZK(); + CancellationCode killPartMoveToShard(const UUID & task_uuid); - /// We could have one thread per Entry and worry about concurrency issues. - /// Or we could have a single thread trying to run one step at a time. - bool step(); - - std::vector getEntries() const; + std::vector getEntries(); private: void run(); - void stepEntry(const Entry & entry, zkutil::ZooKeeperPtr zk); + bool step(); + Entry stepEntry(Entry entry, zkutil::ZooKeeperPtr zk); + + Entry getEntryByUUID(const UUID & task_uuid); + void removePins(const Entry & entry, zkutil::ZooKeeperPtr zk); + void syncStateFromZK(); private: StorageReplicatedMergeTree & storage; @@ -153,7 +184,7 @@ private: BackgroundSchedulePool::TaskHolder task; mutable std::mutex state_mutex; - std::map entries; + std::vector entries; public: String entries_znode_path; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 335302e5dda..03bef4581c1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4669,7 +4669,7 @@ void StorageReplicatedMergeTree::alter( /// If new version returns ordinary name, else returns part name containing the first and last month of the month /// NOTE: use it in pair with getFakePartCoveringAllPartsInPartition(...) -static String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, const MergeTreePartInfo & part_info) +String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, const MergeTreePartInfo & part_info) { if (format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { @@ -6568,6 +6568,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( dst_pins.part_uuids.insert(part->uuid); PartMovesBetweenShardsOrchestrator::Entry part_move_entry; + part_move_entry.state = PartMovesBetweenShardsOrchestrator::EntryState::SYNC_SOURCE; part_move_entry.create_time = std::time(nullptr); part_move_entry.update_time = part_move_entry.create_time; part_move_entry.task_uuid = UUIDHelpers::generateV4(); @@ -6591,10 +6592,13 @@ void StorageReplicatedMergeTree::movePartitionToShard( String task_znode_path = dynamic_cast(*responses.back()).path_created; LOG_DEBUG(log, "Created task for part movement between shards at " + task_znode_path); - /// Force refresh local state. This will make the task immediately visible in `system.part_moves_between_shards` table. - part_moves_between_shards_orchestrator.fetchStateFromZK(); + /// TODO(nv): Nice to have support for `replication_alter_partitions_sync`. + /// For now use the system.part_moves_between_shards table for status. +} - // TODO: Add support for `replication_alter_partitions_sync`. +CancellationCode StorageReplicatedMergeTree::killPartMoveToShard(const UUID & task_uuid) +{ + return part_moves_between_shards_orchestrator.killPartMoveToShard(task_uuid); } void StorageReplicatedMergeTree::getCommitPartOps( diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index e76a54ac5ec..9eaf3c318dd 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -682,6 +682,7 @@ private: void replacePartitionFrom(const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr query_context) override; void movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) override; void movePartitionToShard(const ASTPtr & partition, bool move_part, const String & to, ContextPtr query_context) override; + CancellationCode killPartMoveToShard(const UUID & task_uuid) override; void fetchPartition( const ASTPtr & partition, const StorageMetadataPtr & metadata_snapshot, @@ -745,6 +746,8 @@ protected: bool allow_renaming_); }; +String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, const MergeTreePartInfo & part_info); + /** There are three places for each part, where it should be * 1. In the RAM, data_parts, all_data_parts. diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp index 70bc473e241..c397392e9fb 100644 --- a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp +++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp @@ -30,10 +30,12 @@ NamesAndTypesList StorageSystemPartMovesBetweenShards::getNamesAndTypes() { "part_name", std::make_shared() }, { "part_uuid", std::make_shared() }, { "to_shard", std::make_shared() }, + { "dst_part_name", std::make_shared() }, /// Processing status of item. { "update_time", std::make_shared() }, { "state", std::make_shared() }, + { "rollback", std::make_shared() }, { "num_tries", std::make_shared() }, { "last_exception", std::make_shared() }, }; @@ -122,11 +124,13 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, res_columns[col_num++]->insert(entry.part_name); res_columns[col_num++]->insert(entry.part_uuid); res_columns[col_num++]->insert(entry.to_shard); + res_columns[col_num++]->insert(entry.dst_part_name); /// Processing status of item. res_columns[col_num++]->insert(entry.update_time); res_columns[col_num++]->insert(entry.state.toString()); - res_columns[col_num++]->insert(0); + res_columns[col_num++]->insert(entry.rollback); + res_columns[col_num++]->insert(entry.num_tries); res_columns[col_num++]->insert(entry.last_exception_msg); } } diff --git a/tests/integration/test_part_moves_between_shards/test.py b/tests/integration/test_part_moves_between_shards/test.py index 00407f95389..ed7640e5f9e 100644 --- a/tests/integration/test_part_moves_between_shards/test.py +++ b/tests/integration/test_part_moves_between_shards/test.py @@ -1,32 +1,38 @@ -import random -import time - import pytest +import random +import threading +import time from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV +transient_ch_errors = [23, 32, 210] + cluster = ClickHouseCluster(__file__) s0r0 = cluster.add_instance( 's0r0', main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + stay_alive=True, with_zookeeper=True) s0r1 = cluster.add_instance( 's0r1', main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + stay_alive=True, with_zookeeper=True) s1r0 = cluster.add_instance( 's1r0', main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + stay_alive=True, with_zookeeper=True) s1r1 = cluster.add_instance( 's1r1', main_configs=['configs/remote_servers.xml', 'configs/merge_tree.xml'], + stay_alive=True, with_zookeeper=True) @@ -43,12 +49,14 @@ def test_move(started_cluster): for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): for replica_ix, r in enumerate(rs): r.query(""" + DROP TABLE IF EXISTS test_move; CREATE TABLE test_move(v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_move', '{}') ORDER BY tuple() - """.format(shard_ix, replica_ix)) + """.format(shard_ix, r.name)) s0r0.query("SYSTEM STOP MERGES test_move") + s0r1.query("SYSTEM STOP MERGES test_move") s0r0.query("INSERT INTO test_move VALUES (1)") s0r0.query("INSERT INTO test_move VALUES (2)") @@ -63,14 +71,7 @@ def test_move(started_cluster): s0r0.query("SYSTEM START MERGES test_move") s0r0.query("OPTIMIZE TABLE test_move FINAL") - while True: - time.sleep(3) - - print(s0r0.query("SELECT * FROM system.part_moves_between_shards")) - - # Eventually. - if "DONE" == s0r0.query("SELECT state FROM system.part_moves_between_shards WHERE table = 'test_move'").strip(): - break + wait_for_state("DONE", s0r0, "test_move") for n in [s0r0, s0r1]: assert "1" == n.query("SELECT count() FROM test_move").strip() @@ -81,14 +82,7 @@ def test_move(started_cluster): # Move part back s1r0.query("ALTER TABLE test_move MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/test_move'") - while True: - time.sleep(3) - - print(s1r0.query("SELECT * FROM system.part_moves_between_shards")) - - # Eventually. - if "DONE" == s1r0.query("SELECT state FROM system.part_moves_between_shards WHERE table = 'test_move'").strip(): - break + wait_for_state("DONE", s1r0, "test_move") for n in [s0r0, s0r1]: assert "2" == n.query("SELECT count() FROM test_move").strip() @@ -101,17 +95,20 @@ def test_deduplication_while_move(started_cluster): for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): for replica_ix, r in enumerate(rs): r.query(""" + DROP TABLE IF EXISTS test_deduplication; CREATE TABLE test_deduplication(v UInt64) ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_deduplication', '{}') ORDER BY tuple() - """.format(shard_ix, replica_ix)) + """.format(shard_ix, r.name)) r.query(""" - CREATE TABLE t_d AS test_deduplication + DROP TABLE IF EXISTS test_deduplication_d; + CREATE TABLE test_deduplication_d AS test_deduplication ENGINE Distributed('test_cluster', '', test_deduplication) """) s0r0.query("SYSTEM STOP MERGES test_deduplication") + s0r1.query("SYSTEM STOP MERGES test_deduplication") s0r0.query("INSERT INTO test_deduplication VALUES (1)") s0r0.query("INSERT INTO test_deduplication VALUES (2)") @@ -120,7 +117,8 @@ def test_deduplication_while_move(started_cluster): assert "2" == s0r0.query("SELECT count() FROM test_deduplication").strip() assert "0" == s1r0.query("SELECT count() FROM test_deduplication").strip() - s0r0.query("ALTER TABLE test_deduplication MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_deduplication'") + s0r0.query( + "ALTER TABLE test_deduplication MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_deduplication'") s0r0.query("SYSTEM START MERGES test_deduplication") expected = """ @@ -128,32 +126,363 @@ def test_deduplication_while_move(started_cluster): 2 """ - # Verify that we get consisntent result at all times while the part is moving from one shard to another. - while "DONE" != s0r0.query("SELECT state FROM system.part_moves_between_shards WHERE table = 'test_deduplication' ORDER BY create_time DESC LIMIT 1").strip(): + def deduplication_invariant_test(): n = random.choice(list(started_cluster.instances.values())) + assert TSV( + n.query("SELECT * FROM test_deduplication_d ORDER BY v", + settings={"allow_experimental_query_deduplication": 1}) + ) == TSV(expected) - assert TSV(n.query("SELECT * FROM t_d ORDER BY v", settings={ - "allow_experimental_query_deduplication": 1 - })) == TSV(expected) + deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test) + deduplication_invariant.start() + + wait_for_state("DONE", s0r0, "test_deduplication") + + deduplication_invariant.stop_and_assert_no_exception() + + +def test_part_move_step_by_step(started_cluster): + for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): + for replica_ix, r in enumerate(rs): + r.query(""" + DROP TABLE IF EXISTS test_part_move_step_by_step; + CREATE TABLE test_part_move_step_by_step(v UInt64) + ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_part_move_step_by_step', '{}') + ORDER BY tuple() + """.format(shard_ix, r.name)) + + r.query(""" + DROP TABLE IF EXISTS test_part_move_step_by_step_d; + CREATE TABLE test_part_move_step_by_step_d AS test_part_move_step_by_step + ENGINE Distributed('test_cluster', currentDatabase(), test_part_move_step_by_step) + """) + + s0r0.query("SYSTEM STOP MERGES test_part_move_step_by_step") + s0r1.query("SYSTEM STOP MERGES test_part_move_step_by_step") + + s0r0.query("INSERT INTO test_part_move_step_by_step VALUES (1)") + s0r0.query("INSERT INTO test_part_move_step_by_step VALUES (2)") + s0r1.query("SYSTEM SYNC REPLICA test_part_move_step_by_step", timeout=20) + + assert "2" == s0r0.query("SELECT count() FROM test_part_move_step_by_step").strip() + assert "0" == s1r0.query("SELECT count() FROM test_part_move_step_by_step").strip() + + expected = """ +1 +2 +""" + + def deduplication_invariant_test(): + n = random.choice(list(started_cluster.instances.values())) + try: + assert TSV( + n.query("SELECT * FROM test_part_move_step_by_step_d ORDER BY v", + settings={"allow_experimental_query_deduplication": 1}) + ) == TSV(expected) + except QueryRuntimeException as e: + # ignore transient errors that are caused by us restarting nodes + if e.returncode not in transient_ch_errors: + raise e + + deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test) + deduplication_invariant.start() + + # Stop a source replica to prevent SYNC_SOURCE succeeding. + s0r1.stop_clickhouse() + + s0r0.query( + "ALTER TABLE test_part_move_step_by_step MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step'") + + # Should hang on SYNC_SOURCE until all source replicas acknowledge new pinned UUIDs. + wait_for_state("SYNC_SOURCE", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Start all replicas in source shard but stop a replica in destination shard + # to prevent SYNC_DESTINATION succeeding. + s1r1.stop_clickhouse() + s0r1.start_clickhouse() + + # After SYNC_SOURCE step no merges will be assigned. + s0r0.query("SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;") + s0r1.query("SYSTEM START MERGES test_part_move_step_by_step; OPTIMIZE TABLE test_part_move_step_by_step;") + + wait_for_state("SYNC_DESTINATION", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Start previously stopped replica in destination shard to let SYNC_DESTINATION + # succeed. + # Stop the other replica in destination shard to prevent DESTINATION_FETCH succeed. + s1r0.stop_clickhouse() + s1r1.start_clickhouse() + wait_for_state("DESTINATION_FETCH", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Start previously stopped replica in destination shard to let DESTINATION_FETCH + # succeed. + # Stop the other replica in destination shard to prevent DESTINATION_ATTACH succeed. + s1r1.stop_clickhouse() + s1r0.start_clickhouse() + wait_for_state("DESTINATION_ATTACH", s0r0, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Start all replicas in destination shard to let DESTINATION_ATTACH succeed. + # Stop a source replica to prevent SOURCE_DROP succeeding. + s0r0.stop_clickhouse() + s1r1.start_clickhouse() + wait_for_state("SOURCE_DROP", s0r1, "test_part_move_step_by_step", "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + s0r0.start_clickhouse() + wait_for_state("DONE", s0r1, "test_part_move_step_by_step") + deduplication_invariant.assert_no_exception() + + # No hung tasks in replication queue. Would timeout otherwise. + for instance in started_cluster.instances.values(): + instance.query("SYSTEM SYNC REPLICA test_part_move_step_by_step") + + assert "1" == s0r0.query("SELECT count() FROM test_part_move_step_by_step").strip() + assert "1" == s1r0.query("SELECT count() FROM test_part_move_step_by_step").strip() + + deduplication_invariant.stop_and_assert_no_exception() + + +def test_part_move_step_by_step_kill(started_cluster): + for shard_ix, rs in enumerate([[s0r0, s0r1], [s1r0, s1r1]]): + for replica_ix, r in enumerate(rs): + r.query(""" + DROP TABLE IF EXISTS test_part_move_step_by_step_kill; + CREATE TABLE test_part_move_step_by_step_kill(v UInt64) + ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/test_part_move_step_by_step_kill', '{}') + ORDER BY tuple() + """.format(shard_ix, r.name)) + + r.query(""" + DROP TABLE IF EXISTS test_part_move_step_by_step_kill_d; + CREATE TABLE test_part_move_step_by_step_kill_d AS test_part_move_step_by_step_kill + ENGINE Distributed('test_cluster', currentDatabase(), test_part_move_step_by_step_kill) + """) + + s0r0.query("SYSTEM STOP MERGES test_part_move_step_by_step_kill") + s0r1.query("SYSTEM STOP MERGES test_part_move_step_by_step_kill") + + s0r0.query("INSERT INTO test_part_move_step_by_step_kill VALUES (1)") + s0r0.query("INSERT INTO test_part_move_step_by_step_kill VALUES (2)") + s0r1.query("SYSTEM SYNC REPLICA test_part_move_step_by_step_kill", timeout=20) + + assert "2" == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + assert "0" == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + + expected = """ +1 +2 +""" + + def deduplication_invariant_test(): + n = random.choice(list(started_cluster.instances.values())) + try: + assert TSV( + n.query("SELECT * FROM test_part_move_step_by_step_kill_d ORDER BY v", + settings={ + "allow_experimental_query_deduplication": 1 + }) + ) == TSV(expected) + except QueryRuntimeException as e: + # ignore transient errors that are caused by us restarting nodes + if e.returncode not in transient_ch_errors: + raise e + + deduplication_invariant = ConcurrentInvariant(deduplication_invariant_test) + deduplication_invariant.start() + + # Stop a source replica to prevent SYNC_SOURCE succeeding. + s0r1.stop_clickhouse() + + s0r0.query( + "ALTER TABLE test_part_move_step_by_step_kill MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/test_part_move_step_by_step_kill'") + + # Should hang on SYNC_SOURCE until all source replicas acknowledge new pinned UUIDs. + wait_for_state("SYNC_SOURCE", s0r0, "test_part_move_step_by_step_kill", "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Start all replicas in source shard but stop a replica in destination shard + # to prevent SYNC_DESTINATION succeeding. + s1r1.stop_clickhouse() + s0r1.start_clickhouse() + + # After SYNC_SOURCE step no merges will be assigned. + s0r0.query("SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;") + s0r1.query("SYSTEM START MERGES test_part_move_step_by_step_kill; OPTIMIZE TABLE test_part_move_step_by_step_kill;") + + wait_for_state("SYNC_DESTINATION", s0r0, "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Start previously stopped replica in destination shard to let SYNC_DESTINATION + # succeed. + # Stop the other replica in destination shard to prevent DESTINATION_FETCH succeed. + s1r0.stop_clickhouse() + s1r1.start_clickhouse() + wait_for_state("DESTINATION_FETCH", s0r0, "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event") + + # Start previously stopped replica in destination shard to let DESTINATION_FETCH + # succeed. + # Stop the other replica in destination shard to prevent DESTINATION_ATTACH succeed. + s1r1.stop_clickhouse() + s1r0.start_clickhouse() + wait_for_state("DESTINATION_ATTACH", s0r0, "test_part_move_step_by_step_kill", + "Some replicas haven\\'t processed event") + deduplication_invariant.assert_no_exception() + + # Rollback here. + s0r0.query(""" + KILL PART_MOVE_TO_SHARD + WHERE task_uuid = (SELECT task_uuid FROM system.part_moves_between_shards WHERE table = 'test_part_move_step_by_step_kill') + """) + + wait_for_state("DESTINATION_ATTACH", s0r0, "test_part_move_step_by_step_kill", + assert_exception_msg="Some replicas haven\\'t processed event", + assert_rollback=True) + + s1r1.start_clickhouse() + + wait_for_state("CANCELLED", s0r0, "test_part_move_step_by_step_kill", assert_rollback=True) + deduplication_invariant.assert_no_exception() + + # No hung tasks in replication queue. Would timeout otherwise. + for instance in started_cluster.instances.values(): + instance.query("SYSTEM SYNC REPLICA test_part_move_step_by_step_kill") + + assert "2" == s0r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + assert "0" == s1r0.query("SELECT count() FROM test_part_move_step_by_step_kill").strip() + + deduplication_invariant.stop_and_assert_no_exception() def test_move_not_permitted(started_cluster): + # Verify that invariants for part compatibility are checked. + + # Tests are executed in order. Make sure cluster is up if previous test + # failed. + s0r0.start_clickhouse() + s1r0.start_clickhouse() + for ix, n in enumerate([s0r0, s1r0]): - n.query("DROP TABLE IF EXISTS not_permitted") n.query(""" - CREATE TABLE not_permitted(v_{} UInt64) - ENGINE ReplicatedMergeTree('/clickhouse/shard_{}/tables/not_permitted', 'r') - ORDER BY tuple() - """.format(ix, ix)) + DROP TABLE IF EXISTS not_permitted_columns; + + CREATE TABLE not_permitted_columns(v_{ix} UInt64) + ENGINE ReplicatedMergeTree('/clickhouse/shard_{ix}/tables/not_permitted_columns', 'r') + ORDER BY tuple(); + """.format(ix=ix)) - s0r0.query("INSERT INTO not_permitted VALUES (1)") + partition = "date" + if ix > 0: + partition = "v" - with pytest.raises(QueryRuntimeException) as exc: - s0r0.query("ALTER TABLE not_permitted MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted'") + n.query(""" + DROP TABLE IF EXISTS not_permitted_partition; + CREATE TABLE not_permitted_partition(date Date, v UInt64) + ENGINE ReplicatedMergeTree('/clickhouse/shard_{ix}/tables/not_permitted_partition', 'r') + PARTITION BY ({partition}) + ORDER BY tuple(); + """.format(ix=ix, partition=partition)) - assert "DB::Exception: Table columns structure in ZooKeeper is different from local table structure." in str(exc.value) + s0r0.query("INSERT INTO not_permitted_columns VALUES (1)") + s0r0.query("INSERT INTO not_permitted_partition VALUES ('2021-09-03', 1)") - with pytest.raises(QueryRuntimeException) as exc: - s0r0.query("ALTER TABLE not_permitted MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/not_permitted'") + with pytest.raises(QueryRuntimeException, match="DB::Exception: Source and destination are the same"): + s0r0.query("ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_0/tables/not_permitted_columns'") - assert "DB::Exception: Source and destination are the same" in str(exc.value) + with pytest.raises(QueryRuntimeException, match="DB::Exception: Table columns structure in ZooKeeper is different from local table structure."): + s0r0.query("ALTER TABLE not_permitted_columns MOVE PART 'all_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_columns'") + + with pytest.raises(QueryRuntimeException, match="DB::Exception: Existing table metadata in ZooKeeper differs in partition key expression."): + s0r0.query("ALTER TABLE not_permitted_partition MOVE PART '20210903_0_0_0' TO SHARD '/clickhouse/shard_1/tables/not_permitted_partition'") + + +def wait_for_state(desired_state, instance, test_table, assert_exception_msg=None, assert_rollback=False): + last_debug_print_time = time.time() + + print("Waiting to reach state: {}".format(desired_state)) + if assert_exception_msg: + print(" with exception contents: {}".format(assert_exception_msg)) + if assert_rollback: + print(" and rollback: {}".format(assert_rollback)) + + while True: + tasks = TSV.toMat(instance.query( + "SELECT state, num_tries, last_exception, rollback FROM system.part_moves_between_shards WHERE table = '{}'".format( + test_table))) + assert len(tasks) == 1, "only one task expected in this test" + + if time.time() - last_debug_print_time > 30: + last_debug_print_time = time.time() + print("Current state: ", tasks) + + [state, num_tries, last_exception, rollback] = tasks[0] + + if state == desired_state: + if assert_exception_msg and int(num_tries) < 3: + # Let the task be retried a few times when expecting an exception + # to make sure the exception is persistent and the code doesn't + # accidentally continue to run when we expect it not to. + continue + + if assert_exception_msg: + assert assert_exception_msg in last_exception + + if assert_rollback: + assert int(rollback) == 1, "rollback bit isn't set" + + break + elif state in ["DONE", "CANCELLED"]: + raise Exception("Reached terminal state {}, but was waiting for {}".format(state, desired_state)) + + time.sleep(0.1) + + +class ConcurrentInvariant: + def __init__(self, invariant_test, loop_sleep=0.1): + self.invariant_test = invariant_test + self.loop_sleep = loop_sleep + + self.started = False + self.exiting = False + self.exception = None + self.thread = threading.Thread(target=self._loop) + + def start(self): + if self.started: + raise Exception('invariant thread already started') + + self.started = True + self.thread.start() + + def stop_and_assert_no_exception(self): + self._assert_started() + + self.exiting = True + self.thread.join() + + if self.exception: + raise self.exception + + def assert_no_exception(self): + self._assert_started() + + if self.exception: + raise self.exception + + def _loop(self): + try: + while not self.exiting: + self.invariant_test() + time.sleep(self.loop_sleep) + except Exception as e: + self.exiting = True + self.exception = e + + def _assert_started(self): + if not self.started: + raise Exception('invariant thread not started, forgot to call start?') From 008c3c812b09486894b1f6f617cff4de52364ce1 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Mon, 20 Sep 2021 19:08:59 +0200 Subject: [PATCH 006/173] Allow non-replicated ALTER TABLE FETCH/ATTACH in Replicated databases `ALTER TABLE ... FETCH` and `ALTER TABLE ... ATTACH` queries were disabled in the Replicated database engine, because it could cause accidental duplication of data. This enables these queries but without replicating them. In the case of `FETCH`, the part will only be fetched on the server where the query is issued. Similarly, in the case of `ATTACH`, the attached part only needs to be available on the server where the query is issued. If the table itself is using one of the Replicated MergeTree engines, the attached data is then replicated by the table engine itself, without intervention of the database engine. This change is meant to help with live backup/restore when using the Replicated database engine, using FREEZE for backup and ATTACH for restore. --- .../en/engines/database-engines/replicated.md | 2 + src/Interpreters/InterpreterAlterQuery.cpp | 4 +- src/Parsers/ASTAlterQuery.cpp | 11 +++++ src/Parsers/ASTAlterQuery.h | 4 ++ .../test_replicated_database/test.py | 49 ++++++++++++++++++- 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/database-engines/replicated.md b/docs/en/engines/database-engines/replicated.md index 5ce73604207..67671e11cab 100644 --- a/docs/en/engines/database-engines/replicated.md +++ b/docs/en/engines/database-engines/replicated.md @@ -35,6 +35,8 @@ The [system.clusters](../../operations/system-tables/clusters.md) system table c When creating a new replica of the database, this replica creates tables by itself. If the replica has been unavailable for a long time and has lagged behind the replication log — it checks its local metadata with the current metadata in ZooKeeper, moves the extra tables with data to a separate non-replicated database (so as not to accidentally delete anything superfluous), creates the missing tables, updates the table names if they have been renamed. The data is replicated at the `ReplicatedMergeTree` level, i.e. if the table is not replicated, the data will not be replicated (the database is responsible only for metadata). +[`ALTER TABLE FETCH`](../../sql-reference/statements/alter/partition.md) and [`ALTER TABLE ATTACH`](../../sql-reference/statements/alter/partition.md) queries are allowed but not replicated. The database engine will only add the partition/part to the current replica. However, if the table itself uses a Replicated table engine, then the data will be replicated after using `ATTACH`. + ## Usage Example {#usage-example} Creating a cluster with three hosts: diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 08b9846b325..e2b2d487be5 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -55,7 +55,9 @@ BlockIO InterpreterAlterQuery::execute() DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); if (typeid_cast(database.get()) - && !getContext()->getClientInfo().is_replicated_database_internal) + && !getContext()->getClientInfo().is_replicated_database_internal + && !alter.isAttachAlter() + && !alter.isFetchAlter()) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 4b70f25f5c5..140ef972eeb 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -450,6 +450,17 @@ bool ASTAlterQuery::isFreezeAlter() const || isOneCommandTypeOnly(ASTAlterCommand::UNFREEZE_PARTITION) || isOneCommandTypeOnly(ASTAlterCommand::UNFREEZE_ALL); } +bool ASTAlterQuery::isAttachAlter() const +{ + return isOneCommandTypeOnly(ASTAlterCommand::ATTACH_PARTITION); +} + +bool ASTAlterQuery::isFetchAlter() const +{ + return isOneCommandTypeOnly(ASTAlterCommand::FETCH_PARTITION); +} + + /** Get the text that identifies this element. */ String ASTAlterQuery::getID(char delim) const { diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index a5410e797ce..432f0c887fd 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -218,6 +218,10 @@ public: bool isFreezeAlter() const; + bool isAttachAlter() const; + + bool isFetchAlter() const; + String getID(char) const override; ASTPtr clone() const override; diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 450caafb9ba..ff400f4466b 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1,3 +1,5 @@ +import os +import shutil import time import re import pytest @@ -16,7 +18,7 @@ snapshot_recovering_node = cluster.add_instance('snapshot_recovering_node', main all_nodes = [main_node, dummy_node, competing_node, snapshotting_node, snapshot_recovering_node] -uuid_regex = re.compile("[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12}") +uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") def assert_create_query(nodes, table_name, expected): replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) query = "show create table {}".format(table_name) @@ -100,6 +102,51 @@ def test_simple_alter_table(started_cluster, engine): assert_create_query([main_node, dummy_node, competing_node], name, expected) +def get_table_uuid(database, name): + return main_node.query(f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'").strip() + + +@pytest.fixture(scope="module", name="attachable_part") +def fixture_attachable_part(started_cluster): + main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") + main_node.query(f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)") + main_node.query(f"INSERT INTO testdb_attach_atomic.test VALUES (123)") + main_node.query(f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'") + table_uuid = get_table_uuid("testdb_attach_atomic", "test") + return os.path.join(main_node.path, f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0") + + + +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +def test_alter_attach(started_cluster, attachable_part, engine): + name = "alter_attach_test_{}".format(engine) + main_node.query(f"CREATE TABLE testdb.{name} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + table_uuid = get_table_uuid("testdb", name) + # Provide and attach a part to the main node + shutil.copytree( + attachable_part, os.path.join(main_node.path, f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0") + ) + main_node.query(f"ALTER TABLE testdb.{name} ATTACH PART 'all_1_1_0'") + # On the main node, data is attached + assert main_node.query(f"SELECT CounterID FROM testdb.{name}") == "123\n" + # On the other node, data is replicated only if using a Replicated table engine + if engine == "ReplicatedMergeTree": + assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "123\n" + else: + assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "" + + +def test_alter_fetch(started_cluster): + main_node.query("CREATE TABLE testdb.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") + main_node.query("CREATE TABLE testdb.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") + main_node.query("INSERT INTO testdb.fetch_source VALUES (123)") + table_uuid = get_table_uuid("testdb", "fetch_source") + main_node.query(f"ALTER TABLE testdb.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' ") + detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='testdb' AND table='fetch_target'" + assert main_node.query(detached_parts_query) == "all_0_0_0\n" + assert dummy_node.query(detached_parts_query) == "" + + def test_alters_from_different_replicas(started_cluster): # test_alters_from_different_replicas competing_node.query("CREATE DATABASE IF NOT EXISTS testdb ENGINE = Replicated('/clickhouse/databases/test1', 'shard1', 'replica3');") From aa3f4003c82b80331468e789e31dfa7cd7ad0420 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Fri, 24 Sep 2021 12:51:52 +0200 Subject: [PATCH 007/173] Fix hanging DDL queries on Replicated database There was a race condition when issuing a DDL query on a replica just after a new replica was added. If the DDL query is issued after the new replica adds itself to the list of replicas, but before the new replica has finished its recovery, then the first replica adds the new replica to the list of replicas to wait to confirm the query was replicated. Meanwhile, the new replica is still in recovery and applies queries from the /metadata snapshot. When it's done, it bumps its log_ptr without marking the corresponding log entries (if any) as finished. The first replica then waits until distributed_ddl_task_timeout expires and wrongly assumes the query was not replicated. The issue is fixed by remembering the max_log_ptr at the exact point where the replica adds itself to the list of replicas, then mark as finished all queries that happened between that max_log_ptr and the max_log_ptr of the metadata snapshot used in recovery. The bug was randomly observed during a downstream test. It can be reproduced more easily by inserting a sleep of a few seconds at the end of createReplicaNodesInZooKeeper, enough to have time to issue a DDL query on the first replica. --- src/Databases/DatabaseReplicated.cpp | 43 +++++++++++++++++++++++++--- src/Databases/DatabaseReplicated.h | 1 + 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index da03eb6aba6..42389941c98 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -298,10 +298,30 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt /// Write host name to replica_path, it will protect from multiple replicas with the same name auto host_id = getHostID(getContext(), db_uuid); - Coordination::Requests ops; - ops.emplace_back(zkutil::makeCreateRequest(replica_path, host_id, zkutil::CreateMode::Persistent)); - ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/log_ptr", "0", zkutil::CreateMode::Persistent)); - current_zookeeper->multi(ops); + for (int attempts = 10; attempts > 0; --attempts) + { + Coordination::Stat stat; + String max_log_ptr_str = current_zookeeper->get(zookeeper_path + "/max_log_ptr", &stat); + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCreateRequest(replica_path, host_id, zkutil::CreateMode::Persistent)); + ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/log_ptr", "0", zkutil::CreateMode::Persistent)); + /// In addition to creating the replica nodes, we record the max_log_ptr at the instant where + /// we declared ourself as an existing replica. We'll need this during recoverLostReplica to + /// notify other nodes that issued new queries while this node was recovering. + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/max_log_ptr", stat.version)); + Coordination::Responses responses; + const auto code = current_zookeeper->tryMulti(ops, responses); + if (code == Coordination::Error::ZOK) + { + max_log_ptr_at_creation = parse(max_log_ptr_str); + break; + } + else if (code == Coordination::Error::ZNODEEXISTS || attempts == 1) + { + /// If its our last attempt, or if the replica already exists, fail immediately. + zkutil::KeeperMultiException::check(code, ops, responses); + } + } createEmptyLogEntry(current_zookeeper); } @@ -613,6 +633,21 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep InterpreterCreateQuery(query_ast, create_query_context).execute(); } + if (max_log_ptr_at_creation != 0) + { + /// If the replica is new and some of the queries applied during recovery + /// where issued after the replica was created, then other nodes might be + /// waiting for this node to notify them that the query was applied. + for (UInt32 ptr = max_log_ptr_at_creation; ptr <= max_log_ptr; ++ptr) + { + auto entry_name = DDLTaskBase::getLogEntryName(ptr); + auto path = fs::path(zookeeper_path) / "log" / entry_name / "finished" / getFullReplicaName(); + auto status = ExecutionStatus(0).serializeText(); + auto res = current_zookeeper->tryCreate(path, status, zkutil::CreateMode::Persistent); + if (res == Coordination::Error::ZOK) + LOG_INFO(log, "Marked recovered {} as finished", entry_name); + } + } current_zookeeper->set(replica_path + "/log_ptr", toString(max_log_ptr)); } diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 1e0daeed07e..849c33dfbcc 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -90,6 +90,7 @@ private: std::atomic_bool is_readonly = true; std::unique_ptr ddl_worker; + UInt32 max_log_ptr_at_creation = 0; mutable ClusterPtr cluster; }; From 14b1c438ccf2dff7fedd4c9b8b1f64d5ba65da04 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Tue, 21 Sep 2021 17:47:48 +0200 Subject: [PATCH 008/173] Enable all tests using ATTACH/FETCH on Replicated database --- .../queries/0_stateless/00626_replace_partition_from_table.sql | 3 +-- .../00626_replace_partition_from_table_zookeeper.sh | 3 +-- tests/queries/0_stateless/00753_alter_attach.sql | 3 +-- tests/queries/0_stateless/00955_test_final_mark.sql | 3 +-- tests/queries/0_stateless/00955_test_final_mark_use.sh | 3 +-- tests/queries/0_stateless/01015_attach_part.sql | 3 +-- tests/queries/0_stateless/01021_only_tuple_columns.sql | 3 +-- .../queries/0_stateless/01060_shutdown_table_after_detach.sql | 3 +-- tests/queries/0_stateless/01130_in_memory_parts_partitons.sql | 3 +-- .../0_stateless/01417_freeze_partition_verbose_zookeeper.sh | 2 +- tests/queries/0_stateless/01451_detach_drop_part.sql | 3 --- .../0_stateless/01451_replicated_detach_drop_part_long.sql | 2 +- .../01650_fetch_patition_with_macro_in_zk_path_long.sql | 3 +-- tests/queries/0_stateless/02009_array_join_partition.sql | 3 --- .../0_stateless/02012_changed_enum_type_non_replicated.sql | 3 +-- .../queries/0_stateless/02012_zookeeper_changed_enum_type.sql | 2 +- .../02012_zookeeper_changed_enum_type_incompatible.sql | 2 +- tests/queries/1_stateful/00054_merge_tree_partitions.sql | 3 --- .../queries/1_stateful/00152_insert_different_granularity.sql | 2 +- 19 files changed, 16 insertions(+), 36 deletions(-) diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table.sql b/tests/queries/0_stateless/00626_replace_partition_from_table.sql index 1e3e07f8274..958d2184c46 100644 --- a/tests/queries/0_stateless/00626_replace_partition_from_table.sql +++ b/tests/queries/0_stateless/00626_replace_partition_from_table.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel DROP TABLE IF EXISTS src; DROP TABLE IF EXISTS dst; diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 7b3f787eeab..d78f93d6bb3 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-replicated-database, no-parallel -# Tag no-replicated-database: Unsupported type of ALTER query +# Tags: zookeeper, no-parallel # Because REPLACE PARTITION does not forces immediate removal of replaced data parts from local filesystem # (it tries to do it as quick as possible, but it still performed in separate thread asynchronously) diff --git a/tests/queries/0_stateless/00753_alter_attach.sql b/tests/queries/0_stateless/00753_alter_attach.sql index a9661d3a304..ca43fb3aeae 100644 --- a/tests/queries/0_stateless/00753_alter_attach.sql +++ b/tests/queries/0_stateless/00753_alter_attach.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel DROP TABLE IF EXISTS alter_attach; CREATE TABLE alter_attach (x UInt64, p UInt8) ENGINE = MergeTree ORDER BY tuple() PARTITION BY p; diff --git a/tests/queries/0_stateless/00955_test_final_mark.sql b/tests/queries/0_stateless/00955_test_final_mark.sql index 3c92d96e9a9..6615c945ef8 100644 --- a/tests/queries/0_stateless/00955_test_final_mark.sql +++ b/tests/queries/0_stateless/00955_test_final_mark.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel SET send_logs_level = 'fatal'; diff --git a/tests/queries/0_stateless/00955_test_final_mark_use.sh b/tests/queries/0_stateless/00955_test_final_mark_use.sh index 8642011b670..2c3219fbffd 100755 --- a/tests/queries/0_stateless/00955_test_final_mark_use.sh +++ b/tests/queries/0_stateless/00955_test_final_mark_use.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: no-replicated-database, no-parallel -# Tag no-replicated-database: Unsupported type of ALTER query +# Tags: no-parallel CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01015_attach_part.sql b/tests/queries/0_stateless/01015_attach_part.sql index 3d2058f757e..6b786bfbab9 100644 --- a/tests/queries/0_stateless/01015_attach_part.sql +++ b/tests/queries/0_stateless/01015_attach_part.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel DROP TABLE IF EXISTS table_01; diff --git a/tests/queries/0_stateless/01021_only_tuple_columns.sql b/tests/queries/0_stateless/01021_only_tuple_columns.sql index a0be59c3120..02db21bc0b2 100644 --- a/tests/queries/0_stateless/01021_only_tuple_columns.sql +++ b/tests/queries/0_stateless/01021_only_tuple_columns.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel CREATE TABLE test ( diff --git a/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql b/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql index 5e0ed8b298d..bfe928d7003 100644 --- a/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql +++ b/tests/queries/0_stateless/01060_shutdown_table_after_detach.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel DROP TABLE IF EXISTS test; CREATE TABLE test Engine = MergeTree ORDER BY number AS SELECT number, toString(rand()) x from numbers(10000000); diff --git a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql index a0e8d24eff6..aa6f281e0eb 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql @@ -1,5 +1,4 @@ --- Tags: no-replicated-database, no-parallel --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: no-parallel DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh index 5b936b3f2ba..01678d1b500 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Tags: zookeeper, no-replicated-database, no-parallel -# Tag no-replicated-database: Unsupported type of ALTER query +# Tag no-replicated-database: Fails due to additional replicas or shards CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01451_detach_drop_part.sql b/tests/queries/0_stateless/01451_detach_drop_part.sql index 0bc7e13d243..a285730e45f 100644 --- a/tests/queries/0_stateless/01451_detach_drop_part.sql +++ b/tests/queries/0_stateless/01451_detach_drop_part.sql @@ -1,6 +1,3 @@ --- Tags: no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query - DROP TABLE IF EXISTS mt_01451; CREATE TABLE mt_01451 (v UInt8) ENGINE = MergeTree() order by tuple(); diff --git a/tests/queries/0_stateless/01451_replicated_detach_drop_part_long.sql b/tests/queries/0_stateless/01451_replicated_detach_drop_part_long.sql index eb4d5e43115..9f9d1db78f1 100644 --- a/tests/queries/0_stateless/01451_replicated_detach_drop_part_long.sql +++ b/tests/queries/0_stateless/01451_replicated_detach_drop_part_long.sql @@ -1,5 +1,5 @@ -- Tags: long, replica, no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tag no-replicated-database: Fails due to additional replicas or shards SET replication_alter_partitions_sync = 2; diff --git a/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql b/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql index f9b9181b5b7..ce48ad6a02a 100644 --- a/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql +++ b/tests/queries/0_stateless/01650_fetch_patition_with_macro_in_zk_path_long.sql @@ -1,5 +1,4 @@ --- Tags: long, no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: long DROP TABLE IF EXISTS test_01640; DROP TABLE IF EXISTS restore_01640; diff --git a/tests/queries/0_stateless/02009_array_join_partition.sql b/tests/queries/0_stateless/02009_array_join_partition.sql index 1b99deec0f9..a78efe96f66 100644 --- a/tests/queries/0_stateless/02009_array_join_partition.sql +++ b/tests/queries/0_stateless/02009_array_join_partition.sql @@ -1,6 +1,3 @@ --- Tags: no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query - CREATE TABLE table_2009_part (`i` Int64, `d` Date, `s` String) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY i; ALTER TABLE table_2009_part ATTACH PARTITION tuple(arrayJoin([0, 1])); -- {serverError 248} diff --git a/tests/queries/0_stateless/02012_changed_enum_type_non_replicated.sql b/tests/queries/0_stateless/02012_changed_enum_type_non_replicated.sql index 5f53984a8e4..50d5bfe0d39 100644 --- a/tests/queries/0_stateless/02012_changed_enum_type_non_replicated.sql +++ b/tests/queries/0_stateless/02012_changed_enum_type_non_replicated.sql @@ -1,5 +1,4 @@ --- Tags: replica, no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tags: replica create table enum_alter_issue (a Enum8('one' = 1, 'two' = 2)) engine = MergeTree() ORDER BY a; insert into enum_alter_issue values ('one'), ('two'); diff --git a/tests/queries/0_stateless/02012_zookeeper_changed_enum_type.sql b/tests/queries/0_stateless/02012_zookeeper_changed_enum_type.sql index 790db4c5c3d..5dc9ef8ca17 100644 --- a/tests/queries/0_stateless/02012_zookeeper_changed_enum_type.sql +++ b/tests/queries/0_stateless/02012_zookeeper_changed_enum_type.sql @@ -1,5 +1,5 @@ -- Tags: zookeeper, no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tag no-replicated-database: Fails due to additional replicas or shards create table enum_alter_issue (a Enum8('one' = 1, 'two' = 2), b Int) engine = ReplicatedMergeTree('/clickhouse/tables/{database}/test_02012/enum_alter_issue', 'r1') diff --git a/tests/queries/0_stateless/02012_zookeeper_changed_enum_type_incompatible.sql b/tests/queries/0_stateless/02012_zookeeper_changed_enum_type_incompatible.sql index 697758be0c8..e236e6d2767 100644 --- a/tests/queries/0_stateless/02012_zookeeper_changed_enum_type_incompatible.sql +++ b/tests/queries/0_stateless/02012_zookeeper_changed_enum_type_incompatible.sql @@ -1,5 +1,5 @@ -- Tags: zookeeper, no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tag no-replicated-database: Fails due to additional replicas or shards drop table if exists enum_alter_issue; create table enum_alter_issue (a Enum16('one' = 1, 'two' = 2), b Int) diff --git a/tests/queries/1_stateful/00054_merge_tree_partitions.sql b/tests/queries/1_stateful/00054_merge_tree_partitions.sql index 91e67ac857a..73020952977 100644 --- a/tests/queries/1_stateful/00054_merge_tree_partitions.sql +++ b/tests/queries/1_stateful/00054_merge_tree_partitions.sql @@ -1,6 +1,3 @@ --- Tags: no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query - DROP TABLE IF EXISTS test.partitions; CREATE TABLE test.partitions (EventDate Date, CounterID UInt32) ENGINE = MergeTree(EventDate, CounterID, 8192); INSERT INTO test.partitions SELECT EventDate + UserID % 365 AS EventDate, CounterID FROM test.hits WHERE CounterID = 1704509; diff --git a/tests/queries/1_stateful/00152_insert_different_granularity.sql b/tests/queries/1_stateful/00152_insert_different_granularity.sql index 5c802642a5c..da99173ed96 100644 --- a/tests/queries/1_stateful/00152_insert_different_granularity.sql +++ b/tests/queries/1_stateful/00152_insert_different_granularity.sql @@ -1,5 +1,5 @@ -- Tags: no-tsan, no-replicated-database --- Tag no-replicated-database: Unsupported type of ALTER query +-- Tag no-replicated-database: Fails due to additional replicas or shards DROP TABLE IF EXISTS fixed_granularity_table; From 045d4f2d3278653b296397e78a49c238978024a9 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Sun, 26 Sep 2021 18:47:24 +0200 Subject: [PATCH 009/173] Don't replicate DROP/DETACH/DROP DETACHED on Replicated database --- .../en/engines/database-engines/replicated.md | 2 +- src/Interpreters/InterpreterAlterQuery.cpp | 3 +- src/Parsers/ASTAlterQuery.cpp | 5 ++ src/Parsers/ASTAlterQuery.h | 2 + .../configs/settings.xml | 1 + .../test_replicated_database/test.py | 51 +++++++++++++++++++ 6 files changed, 62 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/database-engines/replicated.md b/docs/en/engines/database-engines/replicated.md index 67671e11cab..bdc17d32393 100644 --- a/docs/en/engines/database-engines/replicated.md +++ b/docs/en/engines/database-engines/replicated.md @@ -35,7 +35,7 @@ The [system.clusters](../../operations/system-tables/clusters.md) system table c When creating a new replica of the database, this replica creates tables by itself. If the replica has been unavailable for a long time and has lagged behind the replication log — it checks its local metadata with the current metadata in ZooKeeper, moves the extra tables with data to a separate non-replicated database (so as not to accidentally delete anything superfluous), creates the missing tables, updates the table names if they have been renamed. The data is replicated at the `ReplicatedMergeTree` level, i.e. if the table is not replicated, the data will not be replicated (the database is responsible only for metadata). -[`ALTER TABLE FETCH`](../../sql-reference/statements/alter/partition.md) and [`ALTER TABLE ATTACH`](../../sql-reference/statements/alter/partition.md) queries are allowed but not replicated. The database engine will only add the partition/part to the current replica. However, if the table itself uses a Replicated table engine, then the data will be replicated after using `ATTACH`. +[`ALTER TABLE ATTACH|FETCH|DROP|DROP DETACHED|DETACH PARTITION|PART`](../../sql-reference/statements/alter/partition.md) queries are allowed but not replicated. The database engine will only add/fetch/remove the partition/part to the current replica. However, if the table itself uses a Replicated table engine, then the data will be replicated after using `ATTACH`. ## Usage Example {#usage-example} diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index e2b2d487be5..a1d083e298d 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -57,7 +57,8 @@ BlockIO InterpreterAlterQuery::execute() if (typeid_cast(database.get()) && !getContext()->getClientInfo().is_replicated_database_internal && !alter.isAttachAlter() - && !alter.isFetchAlter()) + && !alter.isFetchAlter() + && !alter.isDropPartitionAlter()) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name); guard->releaseTableLock(); diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 140ef972eeb..f3f83258ac8 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -460,6 +460,11 @@ bool ASTAlterQuery::isFetchAlter() const return isOneCommandTypeOnly(ASTAlterCommand::FETCH_PARTITION); } +bool ASTAlterQuery::isDropPartitionAlter() const +{ + return isOneCommandTypeOnly(ASTAlterCommand::DROP_PARTITION) || isOneCommandTypeOnly(ASTAlterCommand::DROP_DETACHED_PARTITION); +} + /** Get the text that identifies this element. */ String ASTAlterQuery::getID(char delim) const diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h index 432f0c887fd..2f53026c8d8 100644 --- a/src/Parsers/ASTAlterQuery.h +++ b/src/Parsers/ASTAlterQuery.h @@ -222,6 +222,8 @@ public: bool isFetchAlter() const; + bool isDropPartitionAlter() const; + String getID(char) const override; ASTPtr clone() const override; diff --git a/tests/integration/test_replicated_database/configs/settings.xml b/tests/integration/test_replicated_database/configs/settings.xml index 7f45502e20d..83e83a47c0b 100644 --- a/tests/integration/test_replicated_database/configs/settings.xml +++ b/tests/integration/test_replicated_database/configs/settings.xml @@ -1,6 +1,7 @@ + 1 1 1 diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index ff400f4466b..fa1e9cf49fa 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -136,6 +136,57 @@ def test_alter_attach(started_cluster, attachable_part, engine): assert dummy_node.query(f"SELECT CounterID FROM testdb.{name}") == "" +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +def test_alter_drop_part(started_cluster, engine): + table = f"alter_drop_{engine}" + part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" + main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") + if engine == "MergeTree": + dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") + main_node.query(f"ALTER TABLE testdb.{table} DROP PART '{part_name}'") + assert main_node.query(f"SELECT CounterID FROM testdb.{table}") == "" + if engine == "ReplicatedMergeTree": + # The DROP operation is still replicated at the table engine level + assert dummy_node.query(f"SELECT CounterID FROM testdb.{table}") == "" + else: + assert dummy_node.query(f"SELECT CounterID FROM testdb.{table}") == "456\n" + + +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +def test_alter_detach_part(started_cluster, engine): + table = f"alter_detach_{engine}" + part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" + main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") + if engine == "MergeTree": + dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") + main_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") + detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='testdb' AND table='{table}'" + assert main_node.query(detached_parts_query) == f"{part_name}\n" + if engine == "ReplicatedMergeTree": + # The detach operation is still replicated at the table engine level + assert dummy_node.query(detached_parts_query) == f"{part_name}\n" + else: + assert dummy_node.query(detached_parts_query) == "" + + +@pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +def test_alter_drop_detached_part(started_cluster, engine): + table = f"alter_drop_detached_{engine}" + part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" + main_node.query(f"CREATE TABLE testdb.{table} (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)") + main_node.query(f"INSERT INTO testdb.{table} VALUES (123)") + main_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") + if engine == "MergeTree": + dummy_node.query(f"INSERT INTO testdb.{table} VALUES (456)") + dummy_node.query(f"ALTER TABLE testdb.{table} DETACH PART '{part_name}'") + main_node.query(f"ALTER TABLE testdb.{table} DROP DETACHED PART '{part_name}'") + detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='testdb' AND table='{table}'" + assert main_node.query(detached_parts_query) == "" + assert dummy_node.query(detached_parts_query) == f"{part_name}\n" + + def test_alter_fetch(started_cluster): main_node.query("CREATE TABLE testdb.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") main_node.query("CREATE TABLE testdb.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)") From 72473e26e2bda8ef019bfdfe95b68e80ba6c9e86 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Wed, 29 Sep 2021 21:08:20 +0300 Subject: [PATCH 010/173] update example --- docs/en/operations/system-tables/replicas.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index 251c7e65537..7bca3e78ff9 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -45,6 +45,9 @@ last_queue_update: 2020-02-20 08:34:32 absolute_delay: 0 total_replicas: 2 active_replicas: 2 +last_queue_update_exception: +zookeeper_exception: +replica_is_active: {'example01-06-1.yandex.ru':1} ``` Columns: @@ -82,8 +85,9 @@ The next 4 columns have a non-zero value only where there is an active session w - `absolute_delay` (`UInt64`) - How big lag in seconds the current replica has. - `total_replicas` (`UInt8`) - The total number of known replicas of this table. - `active_replicas` (`UInt8`) - The number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas). -- `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — Map between replica name and is replica active. - `last_queue_update_exception` (`String`) - When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions. +- `zookeeper_exception` (`String`) - The last exception message, got if the error happened when fetching the info from ZooKeeper. +- `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — Map between replica name and is replica active. If you request all the columns, the table may work a bit slowly, since several reads from ZooKeeper are made for each row. If you do not request the last 4 columns (log_max_index, log_pointer, total_replicas, active_replicas), the table works quickly. From b336947f77feb0fabc82ddc1522d9639f8ee3947 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Fri, 1 Oct 2021 16:06:10 +0300 Subject: [PATCH 011/173] new example --- docs/en/operations/system-tables/replicas.md | 68 ++++++++++---------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index 7bca3e78ff9..c959100b6e4 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -13,41 +13,43 @@ FORMAT Vertical ``` ``` text +Query id: dc6dcbcb-dc28-4df9-ae27-4354f5b3b13e + Row 1: ────── -database: merge -table: visits -engine: ReplicatedCollapsingMergeTree -is_leader: 1 -can_become_leader: 1 -is_readonly: 0 -is_session_expired: 0 -future_parts: 1 -parts_to_check: 0 -zookeeper_path: /clickhouse/tables/01-06/visits -replica_name: example01-06-1.yandex.ru -replica_path: /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru -columns_version: 9 -queue_size: 1 -inserts_in_queue: 0 -merges_in_queue: 1 -part_mutations_in_queue: 0 -queue_oldest_time: 2020-02-20 08:34:30 -inserts_oldest_time: 1970-01-01 00:00:00 -merges_oldest_time: 2020-02-20 08:34:30 -part_mutations_oldest_time: 1970-01-01 00:00:00 -oldest_part_to_get: -oldest_part_to_merge_to: 20200220_20284_20840_7 -oldest_part_to_mutate_to: -log_max_index: 596273 -log_pointer: 596274 -last_queue_update: 2020-02-20 08:34:32 -absolute_delay: 0 -total_replicas: 2 -active_replicas: 2 -last_queue_update_exception: -zookeeper_exception: -replica_is_active: {'example01-06-1.yandex.ru':1} +database: default +table: test +engine: ReplicatedMergeTree +is_leader: 1 +can_become_leader: 1 +is_readonly: 0 +is_session_expired: 0 +future_parts: 0 +parts_to_check: 0 +zookeeper_path: /test/t +replica_name: 1 +replica_path: /test/t/replicas/1 +columns_version: -1 +queue_size: 0 +inserts_in_queue: 0 +merges_in_queue: 0 +part_mutations_in_queue: 0 +queue_oldest_time: 1970-01-01 03:00:00 +inserts_oldest_time: 1970-01-01 03:00:00 +merges_oldest_time: 1970-01-01 03:00:00 +part_mutations_oldest_time: 1970-01-01 03:00:00 +oldest_part_to_get: +oldest_part_to_merge_to: +oldest_part_to_mutate_to: +log_max_index: 0 +log_pointer: 0 +last_queue_update: 1970-01-01 03:00:00 +absolute_delay: 0 +total_replicas: 1 +active_replicas: 1 +last_queue_update_exception: +zookeeper_exception: +replica_is_active: {'1':1} ``` Columns: From 6169b180cb9374acc1ab87e57e7975724cbfaf8c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 4 Oct 2021 17:27:44 +0300 Subject: [PATCH 012/173] Fix backward compatibility after #21196. --- src/Core/Defines.h | 4 +- src/Interpreters/ClientInfo.h | 8 +++ src/Interpreters/InterpreterSelectQuery.cpp | 14 +++++ src/Server/TCPHandler.cpp | 5 ++ .../test_aggregate_fixed_key.py | 61 +++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py diff --git a/src/Core/Defines.h b/src/Core/Defines.h index ce5c9098399..3620c1dfcfe 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -63,7 +63,9 @@ /// Minimum revision with exactly the same set of aggregation methods and rules to select them. /// Two-level (bucketed) aggregation is incompatible if servers are inconsistent in these rules /// (keys will be placed in different buckets and result will not be fully aggregated). -#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54431 +#define DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 54456 +#define DBMS_MIN_MAJOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 21 +#define DBMS_MIN_MINOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD 4 #define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410 #define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE 54405 diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 71570778645..716c66b8b71 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -83,6 +83,14 @@ public: UInt64 client_version_patch = 0; unsigned client_tcp_protocol_version = 0; + /// In case of distributed query, client info for query is actually a client info of client. + /// In order to get a version of server-initiator, use connection_ values. + /// Also for tcp only. + UInt64 connection_client_version_major = 0; + UInt64 connection_client_version_minor = 0; + UInt64 connection_client_version_patch = 0; + unsigned connection_tcp_protocol_version = 0; + /// For http HTTPMethod http_method = HTTPMethod::UNKNOWN; String http_user_agent; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 563e9ba8997..9ef35a774bf 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -75,6 +75,7 @@ #include #include +#include #include #include #include @@ -2609,6 +2610,19 @@ void InterpreterSelectQuery::initSettings() auto & query = getSelectQuery(); if (query.settings()) InterpreterSetQuery(query.settings(), context).executeForCurrentContext(); + + auto & client_info = context->getClientInfo(); + auto min_major = DBMS_MIN_MAJOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD; + auto min_minor = DBMS_MIN_MINOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD; + + if (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && + std::forward_as_tuple(client_info.connection_client_version_major, client_info.connection_client_version_minor) < std::forward_as_tuple(min_major, min_minor)) + { + /// Disable two-level aggregation due to version incompatibility. + context->setSetting("group_by_two_level_threshold", Field(0)); + context->setSetting("group_by_two_level_threshold_bytes", Field(0)); + + } } } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 5415bf96443..a139ab5c100 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -944,6 +944,11 @@ void TCPHandler::receiveHello() client_info.client_version_patch = client_version_patch; client_info.client_tcp_protocol_version = client_tcp_protocol_version; + client_info.connection_client_version_major = client_version_major; + client_info.connection_client_version_minor = client_version_minor; + client_info.connection_client_version_patch = client_version_patch; + client_info.connection_tcp_protocol_version = client_tcp_protocol_version; + is_interserver_mode = (user == USER_INTERSERVER_MARKER); if (is_interserver_mode) { diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py new file mode 100644 index 00000000000..8819be527fd --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -0,0 +1,61 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='21.3', with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server') +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server') + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_two_level_merge(start_cluster): + for node in start_cluster.instances.values(): + node.query( + """ + CREATE TABLE IF NOT EXISTS test_two_level_merge(date Date, zone UInt32, number UInt32) + ENGINE = MergeTree() PARTITION BY toUInt64(number / 1000) ORDER BY tuple(); + + INSERT INTO + test_two_level_merge + SELECT + toDate('2021-09-28') - number / 1000, + 249081628, + number + FROM + numbers(15000); + """ + ) + + # covers only the keys64 method + for node in start_cluster.instances.values(): + print(node.query( + """ + SELECT + throwIf(uniqExact(date) != count(), 'group by is borked') + FROM ( + SELECT + date + FROM + remote('node{1,2}', default.test_two_level_merge) + WHERE + date BETWEEN toDate('2021-09-20') AND toDate('2021-09-28') + AND zone = 249081628 + GROUP by date, zone + ) + SETTINGS + group_by_two_level_threshold = 1, + group_by_two_level_threshold_bytes = 1, + max_threads = 2, + prefer_localhost_replica = 0 + """ + )) From b9f8647353ea9eba64df1fdc782f421d4d737bc5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 6 Oct 2021 11:59:41 +0300 Subject: [PATCH 013/173] Fix build with master --- src/Interpreters/InterpreterKillQueryQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 13108e297f5..df31ed75374 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -355,7 +355,8 @@ BlockIO InterpreterKillQueryQuery::execute() ErrorCodes::ACCESS_DENIED); - res_io.in = std::make_shared(header.cloneWithColumns(std::move(res_columns))); + + res_io.pipeline = QueryPipeline(Pipe(std::make_shared(header.cloneWithColumns(std::move(res_columns))))); break; } From dd067f80cd13d25073d73b5e4433dd85e8a533ae Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Wed, 6 Oct 2021 12:40:24 +0300 Subject: [PATCH 014/173] update RU --- docs/ru/operations/system-tables/replicas.md | 67 +++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/docs/ru/operations/system-tables/replicas.md b/docs/ru/operations/system-tables/replicas.md index ff58355145d..c7bc9c5982e 100644 --- a/docs/ru/operations/system-tables/replicas.md +++ b/docs/ru/operations/system-tables/replicas.md @@ -13,38 +13,43 @@ FORMAT Vertical ``` ``` text +Query id: dc6dcbcb-dc28-4df9-ae27-4354f5b3b13e + Row 1: ────── -database: merge -table: visits -engine: ReplicatedCollapsingMergeTree -is_leader: 1 -can_become_leader: 1 -is_readonly: 0 -is_session_expired: 0 -future_parts: 1 -parts_to_check: 0 -zookeeper_path: /clickhouse/tables/01-06/visits -replica_name: example01-06-1.yandex.ru -replica_path: /clickhouse/tables/01-06/visits/replicas/example01-06-1.yandex.ru -columns_version: 9 -queue_size: 1 -inserts_in_queue: 0 -merges_in_queue: 1 -part_mutations_in_queue: 0 -queue_oldest_time: 2020-02-20 08:34:30 -inserts_oldest_time: 0000-00-00 00:00:00 -merges_oldest_time: 2020-02-20 08:34:30 -part_mutations_oldest_time: 0000-00-00 00:00:00 -oldest_part_to_get: -oldest_part_to_merge_to: 20200220_20284_20840_7 -oldest_part_to_mutate_to: -log_max_index: 596273 -log_pointer: 596274 -last_queue_update: 2020-02-20 08:34:32 -absolute_delay: 0 -total_replicas: 2 -active_replicas: 2 +database: default +table: test +engine: ReplicatedMergeTree +is_leader: 1 +can_become_leader: 1 +is_readonly: 0 +is_session_expired: 0 +future_parts: 0 +parts_to_check: 0 +zookeeper_path: /test/t +replica_name: 1 +replica_path: /test/t/replicas/1 +columns_version: -1 +queue_size: 0 +inserts_in_queue: 0 +merges_in_queue: 0 +part_mutations_in_queue: 0 +queue_oldest_time: 1970-01-01 03:00:00 +inserts_oldest_time: 1970-01-01 03:00:00 +merges_oldest_time: 1970-01-01 03:00:00 +part_mutations_oldest_time: 1970-01-01 03:00:00 +oldest_part_to_get: +oldest_part_to_merge_to: +oldest_part_to_mutate_to: +log_max_index: 0 +log_pointer: 0 +last_queue_update: 1970-01-01 03:00:00 +absolute_delay: 0 +total_replicas: 1 +active_replicas: 1 +last_queue_update_exception: +zookeeper_exception: +replica_is_active: {'1':1} ``` Столбцы: @@ -82,6 +87,8 @@ active_replicas: 2 - `absolute_delay` (`UInt64`) - задержка (в секундах) для текущей реплики. - `total_replicas` (`UInt8`) - общее число известных реплик этой таблицы. - `active_replicas` (`UInt8`) - число реплик этой таблицы, имеющих сессию в ZK; то есть, число работающих реплик. +- `last_queue_update_exception` (`String`) - если в очереди есть битые записи. Особенно важно, когда в ClickHouse нарушается обратная совместимость между версиями, а записи журнала, сделанные более новыми версиями, не могут быть проанализированы старыми версиями. +- `zookeeper_exception` (`String`) - последнее сообщение об исключении. Появляется, если ошибка произошла при получении информации из ZooKeeper. - `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — соответствие между именем реплики и признаком активности реплики. Если запрашивать все столбцы, то таблица может работать слегка медленно, так как на каждую строчку делается несколько чтений из ZK. From ddca7e071292d754cd22a7c22d20bd9c062a45ba Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 6 Oct 2021 12:51:33 +0300 Subject: [PATCH 015/173] Update InterpreterKillQueryQuery.cpp --- src/Interpreters/InterpreterKillQueryQuery.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index df31ed75374..2b949266c17 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -354,8 +354,6 @@ BlockIO InterpreterKillQueryQuery::execute() "Not allowed to kill move partition. To execute this query it's necessary to have the grant " + required_access_rights.toString(), ErrorCodes::ACCESS_DENIED); - - res_io.pipeline = QueryPipeline(Pipe(std::make_shared(header.cloneWithColumns(std::move(res_columns))))); break; From ce645371f5a40862d03b23c973af717255b7620b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 12 Oct 2021 14:46:27 +0300 Subject: [PATCH 016/173] Update container name. --- .../test_backward_compatibility/test_aggregate_fixed_key.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index 8819be527fd..d895ee0eb78 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -3,9 +3,9 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='21.3', with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server') -node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server') +node1 = cluster.add_instance('node1', with_zookeeper=True, image='clickhouse/server', tag='21.3', with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='clickhouse/server') +node3 = cluster.add_instance('node3', with_zookeeper=True, image='clickhouse/server') @pytest.fixture(scope="module") From e9f3cf665199b358e79c6a3b938a5cb25a33fe46 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 18 Oct 2021 17:49:26 +0300 Subject: [PATCH 017/173] Fix used memory calculation --- src/Common/ProgressIndication.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index bf3397f50e1..5a3f8cfc350 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -121,11 +121,9 @@ ProgressIndication::MemoryUsage ProgressIndication::getMemoryUsage() const return std::accumulate(thread_data.cbegin(), thread_data.cend(), MemoryUsage{}, [](MemoryUsage const & acc, auto const & host_data) { - auto host_usage = std::accumulate(host_data.second.cbegin(), host_data.second.cend(), ZERO, - [](UInt64 memory, auto const & data) - { - return memory + data.second.memory_usage; - }); + UInt64 host_usage = 0; + if (auto it = host_data.second.find(ZERO); it != host_data.second.end()) + host_usage = it->second.memory_usage; return MemoryUsage{.total = acc.total + host_usage, .max = std::max(acc.max, host_usage)}; }); } From 89fe606d15f94b95b9633525a9dc0e4d052610ff Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 18 Oct 2021 23:16:02 +0300 Subject: [PATCH 018/173] try fix 'some fetches may stuck' --- src/Storages/MergeTree/MergeTreePartInfo.h | 3 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 58 +++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 14 +- .../ReplicatedMergeTreeRestartingThread.cpp | 2 + src/Storages/StorageReplicatedMergeTree.cpp | 216 +++++++++++++----- 5 files changed, 215 insertions(+), 78 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 82fe0b860c8..b4c5687209b 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -139,7 +139,8 @@ struct DetachedPartInfo : public MergeTreePartInfo "clone", "attaching", "deleting", - "tmp-fetch" + "tmp-fetch", + "covered-by-broken", }); /// NOTE: It may parse part info incorrectly. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f397e73aa31..6c1cd04d4ae 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -55,6 +55,15 @@ void ReplicatedMergeTreeQueue::clear() mutation_pointer.clear(); } +void ReplicatedMergeTreeQueue::setBrokenPartsToEnqueueFetchesOnLoading(Strings && parts_to_fetch) +{ + std::lock_guard lock(state_mutex); + /// Can be called only before queue initialization + assert(broken_parts_to_enqueue_fetches_on_loading.empty()); + assert(virtual_parts.size() == 0); + broken_parts_to_enqueue_fetches_on_loading = std::move(parts_to_fetch); +} + void ReplicatedMergeTreeQueue::initialize(zkutil::ZooKeeperPtr zookeeper) { std::lock_guard lock(state_mutex); @@ -176,6 +185,19 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) } +void ReplicatedMergeTreeQueue::createLogEntriesToFetchBrokenParts() +{ + std::lock_guard lock(state_mutex); + if (broken_parts_to_enqueue_fetches_on_loading.empty()) + return; + + for (const auto & broken_part_name : broken_parts_to_enqueue_fetches_on_loading) + storage.removePartAndEnqueueFetch(broken_part_name); + + broken_parts_to_enqueue_fetches_on_loading.clear(); +} + + void ReplicatedMergeTreeQueue::insertUnlocked( const LogEntryPtr & entry, std::optional & min_unprocessed_insert_time_changed, std::lock_guard & state_lock) @@ -982,7 +1004,7 @@ bool ReplicatedMergeTreeQueue::checkReplaceRangeCanBeRemoved(const MergeTreePart void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( zkutil::ZooKeeperPtr zookeeper, const MergeTreePartInfo & part_info, - const ReplicatedMergeTreeLogEntryData & current) + const std::optional & current) { /// TODO is it possible to simplify it? Queue to_wait; @@ -993,19 +1015,21 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( /// Remove operations with parts, contained in the range to be deleted, from the queue. std::unique_lock lock(state_mutex); - [[maybe_unused]] bool called_from_alter_query_directly = current.replace_range_entry && current.replace_range_entry->columns_version < 0; - assert(currently_executing_drop_or_replace_range || called_from_alter_query_directly); + [[maybe_unused]] bool called_from_alter_query_directly = current && current->replace_range_entry && current->replace_range_entry->columns_version < 0; + [[maybe_unused]] bool called_for_broken_part = !current; + assert(currently_executing_drop_or_replace_range || called_from_alter_query_directly || called_for_broken_part); for (Queue::iterator it = queue.begin(); it != queue.end();) { auto type = (*it)->type; - bool is_simple_producing_op = type == LogEntry::GET_PART || type == LogEntry::ATTACH_PART || type == LogEntry::MERGE_PARTS || type == LogEntry::MUTATE_PART; + bool simple_op_covered = is_simple_producing_op && part_info.contains(MergeTreePartInfo::fromPartName((*it)->new_part_name, format_version)); - if (simple_op_covered || checkReplaceRangeCanBeRemoved(part_info, *it, current)) + bool replace_range_covered = current && checkReplaceRangeCanBeRemoved(part_info, *it, *current); + if (simple_op_covered || replace_range_covered) { if ((*it)->currently_executing) to_wait.push_back(*it); @@ -1035,16 +1059,20 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( } -bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const String & log_entry_name, const String & new_part_name, +bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & entry, const String & new_part_name, String & out_reason, std::lock_guard & /* queue_lock */) const { /// Let's check if the same part is now being created by another action. - if (future_parts.count(new_part_name)) + auto entry_for_same_part_it = future_parts.find(new_part_name); + if (entry_for_same_part_it != future_parts.end()) { - const char * format_str = "Not executing log entry {} for part {} " - "because another log entry for the same part is being processed. This shouldn't happen often."; - LOG_INFO(log, format_str, log_entry_name, new_part_name); - out_reason = fmt::format(format_str, log_entry_name, new_part_name); + const LogEntry & another_entry = *entry_for_same_part_it->second; + const char * format_str = "Not executing log entry {} of type {} for part {} " + "because another log entry {} of type {} for the same part ({}) is being processed. This shouldn't happen often."; + LOG_INFO(log, format_str, entry.znode_name, entry.type, entry.new_part_name, + another_entry.znode_name, another_entry.type, another_entry.new_part_name); + out_reason = fmt::format(format_str, entry.znode_name, entry.type, entry.new_part_name, + another_entry.znode_name, another_entry.type, another_entry.new_part_name); return false; /** When the corresponding action is completed, then `isNotCoveredByFuturePart` next time, will succeed, @@ -1067,8 +1095,8 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const String & log_ { const char * format_str = "Not executing log entry {} for part {} " "because it is covered by part {} that is currently executing."; - LOG_TRACE(log, format_str, log_entry_name, new_part_name, future_part_elem.first); - out_reason = fmt::format(format_str, log_entry_name, new_part_name, future_part_elem.first); + LOG_TRACE(log, format_str, entry.znode_name, new_part_name, future_part_elem.first); + out_reason = fmt::format(format_str, entry.znode_name, new_part_name, future_part_elem.first); return false; } } @@ -1101,7 +1129,7 @@ bool ReplicatedMergeTreeQueue::addFuturePartIfNotCoveredByThem(const String & pa if (drop_ranges.isAffectedByDropRange(part_name, reject_reason)) return false; - if (isNotCoveredByFuturePartsImpl(entry.znode_name, part_name, reject_reason, lock)) + if (isNotCoveredByFuturePartsImpl(entry, part_name, reject_reason, lock)) { CurrentlyExecuting::setActualPartName(entry, part_name, *this); return true; @@ -1122,7 +1150,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( /// some other entry which is currently executing, then we can postpone this entry. for (const String & new_part_name : entry.getVirtualPartNames(format_version)) { - if (!isNotCoveredByFuturePartsImpl(entry.znode_name, new_part_name, out_postpone_reason, state_lock)) + if (!isNotCoveredByFuturePartsImpl(entry, new_part_name, out_postpone_reason, state_lock)) return false; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 30c8bafcf6e..c059ff7a072 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -156,6 +156,8 @@ private: /// We need it because alters have to be executed sequentially (one by one). ReplicatedMergeTreeAltersSequence alter_sequence; + Strings broken_parts_to_enqueue_fetches_on_loading; + /// List of subscribers /// A subscriber callback is called when an entry queue is deleted mutable std::mutex subscribers_mutex; @@ -208,7 +210,7 @@ private: * Should be called under state_mutex. */ bool isNotCoveredByFuturePartsImpl( - const String & log_entry_name, + const LogEntry & entry, const String & new_part_name, String & out_reason, std::lock_guard & state_lock) const; @@ -324,7 +326,8 @@ public: /** Remove the action from the queue with the parts covered by part_name (from ZK and from the RAM). * And also wait for the completion of their execution, if they are now being executed. */ - void removePartProducingOpsInRange(zkutil::ZooKeeperPtr zookeeper, const MergeTreePartInfo & part_info, const ReplicatedMergeTreeLogEntryData & current); + void removePartProducingOpsInRange(zkutil::ZooKeeperPtr zookeeper, const MergeTreePartInfo & part_info, + const std::optional & current); /** In the case where there are not enough parts to perform the merge in part_name * - move actions with merged parts to the end of the queue @@ -455,6 +458,13 @@ public: /// It's needed because queue itself can trigger it's task handler and in /// this case race condition is possible. QueueLocks lockQueue(); + + /// Can be called only on data parts loading. + /// We need loaded queue to create GET_PART entry for broken (or missing) part, + /// but queue is not loaded yet on data parts loading. + void setBrokenPartsToEnqueueFetchesOnLoading(Strings && parts_to_fetch); + /// Must be called right after queue loading. + void createLogEntriesToFetchBrokenParts(); }; class ReplicatedMergeTreeMergePredicate diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 16597fc4be9..c29ae873c5c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -178,6 +178,8 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() storage.queue.load(zookeeper); + storage.queue.createLogEntriesToFetchBrokenParts(); + /// pullLogsToQueue() after we mark replica 'is_active' (and after we repair if it was lost); /// because cleanup_thread doesn't delete log_pointer of active replicas. storage.queue.pullLogsToQueue(zookeeper, {}, ReplicatedMergeTreeQueue::LOAD); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 416d37cd351..6980d523ed5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1211,44 +1211,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) } /// Add to the queue jobs to pick up the missing parts from other replicas and remove from ZK the information that we have them. - std::vector> exists_futures; - exists_futures.reserve(parts_to_fetch.size()); - for (const String & part_name : parts_to_fetch) - { - String part_path = fs::path(replica_path) / "parts" / part_name; - exists_futures.emplace_back(zookeeper->asyncExists(part_path)); - } - - std::vector> enqueue_futures; - enqueue_futures.reserve(parts_to_fetch.size()); - for (size_t i = 0; i < parts_to_fetch.size(); ++i) - { - const String & part_name = parts_to_fetch[i]; - - Coordination::Requests ops; - - LOG_ERROR(log, "Removing locally missing part from ZooKeeper and queueing a fetch: {}", part_name); - time_t part_create_time = 0; - Coordination::ExistsResponse exists_resp = exists_futures[i].get(); - if (exists_resp.error == Coordination::Error::ZOK) - { - part_create_time = exists_resp.stat.ctime / 1000; - removePartFromZooKeeper(part_name, ops, exists_resp.stat.numChildren > 0); - } - LogEntry log_entry; - log_entry.type = LogEntry::GET_PART; - log_entry.source_replica = ""; - log_entry.new_part_name = part_name; - log_entry.create_time = part_create_time; - - /// We assume that this occurs before the queue is loaded (queue.initialize). - ops.emplace_back(zkutil::makeCreateRequest( - fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential)); - enqueue_futures.emplace_back(zookeeper->asyncMulti(ops)); - } - - for (auto & future : enqueue_futures) - future.get(); + queue.setBrokenPartsToEnqueueFetchesOnLoading(std::move(parts_to_fetch)); /// Remove extra local parts. for (const DataPartPtr & part : unexpected_parts) @@ -2343,13 +2306,57 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo std::sort(source_queue_names.begin(), source_queue_names.end()); - Strings source_queue; - for (const String & entry_name : source_queue_names) + struct QueueEntryInfo { - String entry; - if (!zookeeper->tryGet(fs::path(source_path) / "queue" / entry_name, entry)) - continue; - source_queue.push_back(entry); + String data; + Coordination::Stat stat; + LogEntryPtr parsed_entry; + bool need_copy = true; + }; + + std::vector source_queue; + ActiveDataPartSet get_part_set{format_version}; + ActiveDataPartSet drop_range_set{format_version}; + + { + std::vector queue_get_futures; + source_queue.resize(source_queue_names.size()); + queue_get_futures.reserve(source_queue_names.size()); + + for (const String & entry_name : source_queue_names) + queue_get_futures.push_back(zookeeper->asyncTryGet(fs::path(source_path) / "queue" / entry_name)); + for (size_t i = 0; i < source_queue_names.size(); ++i) + { + auto res = queue_get_futures[i].get(); + /// It's ok if entry is already executed and removed: we also will get source parts set. + if (res.error == Coordination::Error::ZNONODE) + continue; + + assert(res.error == Coordination::Error::ZOK); + auto & info = source_queue[i]; + info.data = std::move(res.data); + info.stat = std::move(res.stat); + try + { + info.parsed_entry = LogEntry::parse(info.data, info.stat); + } + catch (...) + { + tryLogCurrentException(log, "Cannot parse source queue entry " + source_queue_names[i]); + } + + /// It may be ok if source replica has newer version. We will copy entry as is. + if (!info.parsed_entry) + continue; + + info.parsed_entry->znode_name = source_queue_names[i]; + + if (info.parsed_entry->type == LogEntry::GET_PART) + get_part_set.add(info.parsed_entry->new_part_name); + + if (info.parsed_entry->type == LogEntry::DROP_RANGE) + drop_range_set.add(info.parsed_entry->new_part_name); + } } /// We should do it after copying queue, because some ALTER_METADATA entries can be lost otherwise. @@ -2357,9 +2364,10 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo /// Add to the queue jobs to receive all the active parts that the reference/master replica has. Strings source_replica_parts = zookeeper->getChildren(fs::path(source_path) / "parts"); - ActiveDataPartSet active_parts_set(format_version, source_replica_parts); + for (const auto & active_part : source_replica_parts) + get_part_set.add(active_part); - Strings active_parts = active_parts_set.getParts(); + Strings active_parts = get_part_set.getParts(); /// Remove local parts if source replica does not have them, because such parts will never be fetched by other replicas. Strings local_parts_in_zk = zookeeper->getChildren(fs::path(replica_path) / "parts"); @@ -2367,7 +2375,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo for (const auto & part : local_parts_in_zk) { - if (active_parts_set.getContainingPart(part).empty()) + if (get_part_set.getContainingPart(part).empty()) { parts_to_remove_from_zk.emplace_back(part); LOG_WARNING(log, "Source replica does not have part {}. Removing it from ZooKeeper.", part); @@ -2392,7 +2400,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo for (const auto & part : local_active_parts) { - if (active_parts_set.getContainingPart(part->name).empty()) + if (get_part_set.getContainingPart(part->name).empty()) { parts_to_remove_from_working_set.emplace_back(part); LOG_WARNING(log, "Source replica does not have part {}. Removing it from working set.", part->name); @@ -2412,13 +2420,54 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo removePartsFromWorkingSet(parts_to_remove_from_working_set, true); + std::unordered_set created_get_parts; + + /// Avoid creation of GET_PART entries which covered by another GET_PART or DROP_RANGE + /// and creation of multiple entries with the same new_part_name. + auto should_ignore_log_entry = [&] (const String & part_name, const String & log_msg_context) -> bool + { + /// We should not create entries covered by DROP_RANGE, because we will remove them anyway (kind of optimization). + String covering_drop_range = drop_range_set.getContainingPart(part_name); + if (!covering_drop_range.empty()) + { + LOG_TRACE(log, "{} {}: it's covered by DROP_RANGE {}", log_msg_context, part_name, covering_drop_range); + return true; + } + + /// We should not create entries covered by GET_PART, + /// because GET_PART entry has no source parts and we can execute it only by fetching. + /// Parts covered by GET_PART are useless and may cause replication to stuck if covered part is lost. + String covering_get_part_entry = get_part_set.getContainingPart(part_name); + + if (covering_get_part_entry.empty()) + return false; + + if (covering_get_part_entry != part_name) + { + LOG_TRACE(log, "{} {}: it's covered by GET_PART {}", log_msg_context, part_name, covering_get_part_entry); + return true; + } + + if (created_get_parts.count(part_name)) + { + /// NOTE It would be better to copy log entry instead of creating GET_PART + /// if there are GET_PART and log entry of other type with the same new_part_name. + /// But it's a bit harder to implement, because it requires full-fledged virtual_parts set. + LOG_TRACE(log, "{} {}: GET_PART for it is already created", log_msg_context, part_name); + return true; + } + + return false; + }; + for (const String & name : active_parts) { + if (should_ignore_log_entry(name, "Not fetching")) + continue; + LogEntry log_entry; - if (!are_restoring_replica) - log_entry.type = LogEntry::GET_PART; - else + if (are_restoring_replica) { LOG_DEBUG(log, "Obtaining checksum for path {}", name); @@ -2443,23 +2492,46 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo const auto [lo, hi] = desired_checksums.hash_of_all_files; log_entry.part_checksum = getHexUIntUppercase(hi) + getHexUIntUppercase(lo); } + else + { + log_entry.type = LogEntry::GET_PART; + } log_entry.source_replica = ""; log_entry.new_part_name = name; log_entry.create_time = tryGetPartCreateTime(zookeeper, source_path, name); + LOG_TEST(log, "Enqueueing {} for fetch", name); zookeeper->create(fs::path(replica_path) / "queue/queue-", log_entry.toString(), zkutil::CreateMode::PersistentSequential); + created_get_parts.insert(name); } - LOG_DEBUG(log, "Queued {} parts to be fetched", active_parts.size()); + size_t total_parts_to_fetch = created_get_parts.size(); + LOG_DEBUG(log, "Queued {} parts to be fetched, {} parts ignored", total_parts_to_fetch, active_parts.size() - total_parts_to_fetch); /// Add content of the reference/master replica queue to the queue. - for (const String & entry : source_queue) + size_t total_entries_to_copy = 0; + for (const auto & entry_info : source_queue) { - zookeeper->create(fs::path(replica_path) / "queue/queue-", entry, zkutil::CreateMode::PersistentSequential); + if (entry_info.parsed_entry && !entry_info.parsed_entry->new_part_name.empty()) + { + const String & part_name = entry_info.parsed_entry->new_part_name; + const String & entry_name = entry_info.parsed_entry->znode_name; + const auto & entry_type = entry_info.parsed_entry->type; + + if (should_ignore_log_entry(part_name, fmt::format("Not copying {} {} ", entry_name, entry_type))) + continue; + + if (entry_info.parsed_entry->type == LogEntry::GET_PART) + created_get_parts.insert(part_name); + } + + LOG_TEST(log, "Copying entry {}", entry_info.data); + zookeeper->create(fs::path(replica_path) / "queue/queue-", entry_info.data, zkutil::CreateMode::PersistentSequential); + ++total_entries_to_copy; } - LOG_DEBUG(log, "Copied {} queue entries", source_queue.size()); + LOG_DEBUG(log, "Copied {} queue entries, {} entries ignored", total_entries_to_copy, source_queue.size() - total_entries_to_copy); } @@ -3196,6 +3268,34 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n { auto zookeeper = getZooKeeper(); + /// We don't know exactly what happened to broken part + /// and we are going to remove all covered log entries. + /// It's quite dangerous, so clone covered parts to detached. + auto broken_part_info = MergeTreePartInfo::fromPartName(part_name, format_version); + + auto partition_range = getDataPartsPartitionRange(broken_part_info.partition_id); + for (const auto & part : partition_range) + { + if (!broken_part_info.contains(part->info)) + continue; + + /// Broken part itself ether already moved to detached or does not exist. + assert(broken_part_info != part->info); + part->makeCloneInDetached("covered-by-broken", getInMemoryMetadataPtr()); + } + + /// It's possible that queue contains entries covered by part_name. + /// For example, we had GET_PART all_1_42_5 and MUTATE_PART all_1_42_5_63, + /// then all_1_42_5_63 was executed by fetching, but part was written to disk incorrectly. + /// In this case we have to remove it as broken and create GET_PART all_1_42_5_63 to fetch it again, + /// but GET_PART all_1_42_5 may be still in the queue. + /// We should remove all covered entries before creating GET_PART entry, because: + /// 1. In the situation described above, we do not know how to merge/mutate all_1_42_5_63 from all_1_42_5, + /// so GET_PART all_1_42_5 (and all source parts) is useless. The only thing we can do is to fetch all_1_42_5_63. + /// 2. If all_1_42_5_63 is lost, then replication may stuck waiting for all_1_42_5_63 to appear, + /// because we may have some covered parts (more precisely, parts with the same min and max blocks) + queue.removePartProducingOpsInRange(zookeeper, broken_part_info, {}); + String part_path = fs::path(replica_path) / "parts" / part_name; Coordination::Requests ops; @@ -3218,11 +3318,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n fs::path(replica_path) / "queue/queue-", log_entry->toString(), zkutil::CreateMode::PersistentSequential)); - auto results = zookeeper->multi(ops); - - String path_created = dynamic_cast(*results.back()).path_created; - log_entry->znode_name = path_created.substr(path_created.find_last_of('/') + 1); - queue.insert(zookeeper, log_entry); + zookeeper->multi(ops); } From caf972db48c38a24c588c5c333016979812ed6b5 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 19 Oct 2021 14:56:04 +0300 Subject: [PATCH 019/173] fix --- src/Storages/StorageReplicatedMergeTree.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6980d523ed5..9a3c88ca83f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2308,10 +2308,9 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo struct QueueEntryInfo { - String data; - Coordination::Stat stat; - LogEntryPtr parsed_entry; - bool need_copy = true; + String data = {}; + Coordination::Stat stat = {}; + LogEntryPtr parsed_entry = {}; }; std::vector source_queue; @@ -2320,11 +2319,12 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo { std::vector queue_get_futures; - source_queue.resize(source_queue_names.size()); queue_get_futures.reserve(source_queue_names.size()); for (const String & entry_name : source_queue_names) queue_get_futures.push_back(zookeeper->asyncTryGet(fs::path(source_path) / "queue" / entry_name)); + + source_queue.reserve(source_queue_names.size()); for (size_t i = 0; i < source_queue_names.size(); ++i) { auto res = queue_get_futures[i].get(); @@ -2333,7 +2333,8 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo continue; assert(res.error == Coordination::Error::ZOK); - auto & info = source_queue[i]; + source_queue.emplace_back(); + auto & info = source_queue.back(); info.data = std::move(res.data); info.stat = std::move(res.stat); try @@ -2448,6 +2449,11 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo return true; } + /// NOTE: It does not completely avoids duplication of GET_PART entries, + /// because it's possible that source replica has executed some GET_PART after we copied it's queue, + /// but before we copied its active parts set. In this case we will GET_PART entry in our queue + /// and later will pull the original GET_PART from replication log. + /// It should not cause any issues, but it does not allow to get rid of duplicated entries and add an assertion. if (created_get_parts.count(part_name)) { /// NOTE It would be better to copy log entry instead of creating GET_PART @@ -2513,6 +2519,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo size_t total_entries_to_copy = 0; for (const auto & entry_info : source_queue) { + assert(!entry_info.data.empty()); if (entry_info.parsed_entry && !entry_info.parsed_entry->new_part_name.empty()) { const String & part_name = entry_info.parsed_entry->new_part_name; From 170a4957ca28862fdf1915c9236b97872a5f0fe2 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 20 Oct 2021 01:31:39 +0300 Subject: [PATCH 020/173] Add normalizeUTF8NFC function --- .../functions/string-functions.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 05c291bb41d..6c5ed485a3d 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -810,6 +810,28 @@ Result: └─────┘ ``` +## normalizeUTF8NFC {#normalizeutf8nfc} + +Converts a string to [NFC normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text. + +**Syntax** + +``` sql +normalizeUTF8NFKD(words) +``` + +**Arguments** + +- `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- NFC normalized string. + +Type: [String](../../sql-reference/data-types/string.md). + + + ## encodeXMLComponent {#encode-xml-component} Escapes characters to place string into XML text node or attribute. From 73f65bbe75db733afb880421f222e95d75c8461f Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 20 Oct 2021 01:42:39 +0300 Subject: [PATCH 021/173] Rewrite Normalization Form C (NFC) --- docs/en/sql-reference/functions/string-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 6c5ed485a3d..4ac3a175aba 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -826,7 +826,7 @@ normalizeUTF8NFKD(words) **Returned value** -- NFC normalized string. +- String transformed to normalization form C (NFC). Type: [String](../../sql-reference/data-types/string.md). From 6b34398e9cd99c7f0b925342356b368d6d17e884 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 20 Oct 2021 17:17:20 +0300 Subject: [PATCH 022/173] support VALUES format in async inserts --- src/Parsers/ParserInsertQuery.cpp | 9 +- src/Processors/Formats/IInputFormat.h | 3 +- .../Formats/Impl/ValuesBlockInputFormat.cpp | 146 ++++++++++-------- .../Formats/Impl/ValuesBlockInputFormat.h | 6 +- .../getSourceFromASTInsertQuery.cpp | 8 +- .../02015_async_insert_7.reference | 6 + .../0_stateless/02015_async_insert_7.sh | 20 +++ 7 files changed, 126 insertions(+), 72 deletions(-) create mode 100644 tests/queries/0_stateless/02015_async_insert_7.reference create mode 100755 tests/queries/0_stateless/02015_async_insert_7.sh diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 6ac6d3b50b9..1bc552af384 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -131,18 +131,22 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } Pos before_values = pos; + String format_str; /// VALUES or FROM INFILE or FORMAT or SELECT if (!infile && s_values.ignore(pos, expected)) { /// If VALUES is defined in query, everything except setting will be parsed as data data = pos->begin; + format_str = "Values"; } else if (s_format.ignore(pos, expected)) { /// If FORMAT is defined, read format name if (!name_p.parse(pos, format, expected)) return false; + + tryGetIdentifierNameInto(format, format_str); } else if (s_select.ignore(pos, expected) || s_with.ignore(pos,expected)) { @@ -155,6 +159,8 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// FORMAT section is expected if we have input() in SELECT part if (s_format.ignore(pos, expected) && !name_p.parse(pos, format, expected)) return false; + + tryGetIdentifierNameInto(format, format_str); } else if (s_watch.ignore(pos, expected)) { @@ -242,9 +248,8 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) tryGetIdentifierNameInto(table, query->table_id.table_name); } - tryGetIdentifierNameInto(format, query->format); - query->columns = columns; + query->format = std::move(format_str); query->select = select; query->watch = watch; query->settings_ast = settings_ast; diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index f133161c3ec..8f5992efb1b 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -55,6 +55,8 @@ public: */ virtual void resetParser(); + virtual void setReadBuffer(ReadBuffer & in_); + virtual const BlockMissingValues & getMissingValues() const { static const BlockMissingValues none; @@ -70,7 +72,6 @@ public: void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } void addBuffer(std::unique_ptr buffer) { owned_buffers.emplace_back(std::move(buffer)); } - void setReadBuffer(ReadBuffer & in_); protected: ColumnMappingPtr column_mapping{}; diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 0f6a21055d0..c15d0d608ee 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -32,13 +32,25 @@ namespace ErrorCodes } -ValuesBlockInputFormat::ValuesBlockInputFormat(ReadBuffer & in_, const Block & header_, const RowInputFormatParams & params_, - const FormatSettings & format_settings_) - : IInputFormat(header_, buf), buf(in_), params(params_), - format_settings(format_settings_), num_columns(header_.columns()), - parser_type_for_column(num_columns, ParserType::Streaming), - attempts_to_deduce_template(num_columns), attempts_to_deduce_template_cached(num_columns), - rows_parsed_using_template(num_columns), templates(num_columns), types(header_.getDataTypes()) +ValuesBlockInputFormat::ValuesBlockInputFormat( + ReadBuffer & in_, + const Block & header_, + const RowInputFormatParams & params_, + const FormatSettings & format_settings_) + : ValuesBlockInputFormat(std::make_unique(in_), header_, params_, format_settings_) +{ +} + +ValuesBlockInputFormat::ValuesBlockInputFormat( + std::unique_ptr buf_, + const Block & header_, + const RowInputFormatParams & params_, + const FormatSettings & format_settings_) + : IInputFormat(header_, *buf_), buf(std::move(buf_)), + params(params_), format_settings(format_settings_), num_columns(header_.columns()), + parser_type_for_column(num_columns, ParserType::Streaming), + attempts_to_deduce_template(num_columns), attempts_to_deduce_template_cached(num_columns), + rows_parsed_using_template(num_columns), templates(num_columns), types(header_.getDataTypes()) { serializations.resize(types.size()); for (size_t i = 0; i < types.size(); ++i) @@ -58,8 +70,8 @@ Chunk ValuesBlockInputFormat::generate() { try { - skipWhitespaceIfAny(buf); - if (buf.eof() || *buf.position() == ';') + skipWhitespaceIfAny(*buf); + if (buf->eof() || *buf->position() == ';') break; readRow(columns, rows_in_block); } @@ -99,12 +111,12 @@ Chunk ValuesBlockInputFormat::generate() void ValuesBlockInputFormat::readRow(MutableColumns & columns, size_t row_num) { - assertChar('(', buf); + assertChar('(', *buf); for (size_t column_idx = 0; column_idx < num_columns; ++column_idx) { - skipWhitespaceIfAny(buf); - PeekableReadBufferCheckpoint checkpoint{buf}; + skipWhitespaceIfAny(*buf); + PeekableReadBufferCheckpoint checkpoint{*buf}; bool read; /// Parse value using fast streaming parser for literals and slow SQL parser for expressions. @@ -123,9 +135,9 @@ void ValuesBlockInputFormat::readRow(MutableColumns & columns, size_t row_num) /// If read is true, value still may be missing. Bit mask for these values will be copied from ConstantExpressionTemplate later. } - skipWhitespaceIfAny(buf); - if (!buf.eof() && *buf.position() == ',') - ++buf.position(); + skipWhitespaceIfAny(*buf); + if (!buf->eof() && *buf->position() == ',') + ++buf->position(); ++total_rows; } @@ -134,7 +146,7 @@ bool ValuesBlockInputFormat::tryParseExpressionUsingTemplate(MutableColumnPtr & { /// Try to parse expression using template if one was successfully deduced while parsing the first row auto settings = context->getSettingsRef(); - if (templates[column_idx]->parseExpression(buf, format_settings, settings)) + if (templates[column_idx]->parseExpression(*buf, format_settings, settings)) { ++rows_parsed_using_template[column_idx]; return true; @@ -154,7 +166,7 @@ bool ValuesBlockInputFormat::tryParseExpressionUsingTemplate(MutableColumnPtr & } /// Do not use this template anymore templates[column_idx].reset(); - buf.rollbackToCheckpoint(); + buf->rollbackToCheckpoint(); /// It will deduce new template or fallback to slow SQL parser return parseExpression(*column, column_idx); @@ -169,13 +181,13 @@ bool ValuesBlockInputFormat::tryReadValue(IColumn & column, size_t column_idx) const auto & type = types[column_idx]; const auto & serialization = serializations[column_idx]; if (format_settings.null_as_default && !type->isNullable()) - read = SerializationNullable::deserializeTextQuotedImpl(column, buf, format_settings, serialization); + read = SerializationNullable::deserializeTextQuotedImpl(column, *buf, format_settings, serialization); else - serialization->deserializeTextQuoted(column, buf, format_settings); + serialization->deserializeTextQuoted(column, *buf, format_settings); rollback_on_exception = true; - skipWhitespaceIfAny(buf); + skipWhitespaceIfAny(*buf); assertDelimiterAfterValue(column_idx); return read; } @@ -190,7 +202,7 @@ bool ValuesBlockInputFormat::tryReadValue(IColumn & column, size_t column_idx) /// Switch to SQL parser and don't try to use streaming parser for complex expressions /// Note: Throwing exceptions for each expression may be very slow because of stacktraces - buf.rollbackToCheckpoint(); + buf->rollbackToCheckpoint(); return parseExpression(column, column_idx); } } @@ -284,11 +296,11 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx /// We need continuous memory containing the expression to use Lexer skipToNextRow(0, 1); - buf.makeContinuousMemoryFromCheckpointToPos(); - buf.rollbackToCheckpoint(); + buf->makeContinuousMemoryFromCheckpointToPos(); + buf->rollbackToCheckpoint(); Expected expected; - Tokens tokens(buf.position(), buf.buffer().end()); + Tokens tokens(buf->position(), buf->buffer().end()); IParser::Pos token_iterator(tokens, settings.max_parser_depth); ASTPtr ast; @@ -302,7 +314,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx if (!parsed) throw Exception("Cannot parse expression of type " + type.getName() + " here: " - + String(buf.position(), std::min(SHOW_CHARS_ON_SYNTAX_ERROR, buf.buffer().end() - buf.position())), + + String(buf->position(), std::min(SHOW_CHARS_ON_SYNTAX_ERROR, buf->buffer().end() - buf->position())), ErrorCodes::SYNTAX_ERROR); ++token_iterator; @@ -316,9 +328,9 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx try { const auto & serialization = serializations[column_idx]; - serialization->deserializeTextQuoted(column, buf, format_settings); + serialization->deserializeTextQuoted(column, *buf, format_settings); rollback_on_exception = true; - skipWhitespaceIfAny(buf); + skipWhitespaceIfAny(*buf); if (checkDelimiterAfterValue(column_idx)) ok = true; } @@ -366,8 +378,8 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx else ++attempts_to_deduce_template[column_idx]; - buf.rollbackToCheckpoint(); - if (templates[column_idx]->parseExpression(buf, format_settings, settings)) + buf->rollbackToCheckpoint(); + if (templates[column_idx]->parseExpression(*buf, format_settings, settings)) { ++rows_parsed_using_template[column_idx]; parser_type_for_column[column_idx] = ParserType::BatchTemplate; @@ -384,9 +396,9 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx std::rethrow_exception(exception); else { - buf.rollbackToCheckpoint(); - size_t len = const_cast(token_iterator->begin) - buf.position(); - throw Exception("Cannot deduce template of expression: " + std::string(buf.position(), len), ErrorCodes::SYNTAX_ERROR); + buf->rollbackToCheckpoint(); + size_t len = const_cast(token_iterator->begin) - buf->position(); + throw Exception("Cannot deduce template of expression: " + std::string(buf->position(), len), ErrorCodes::SYNTAX_ERROR); } } /// Continue parsing without template @@ -397,7 +409,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx throw Exception("Interpreting expressions is disabled", ErrorCodes::SUPPORT_IS_DISABLED); /// Try to evaluate single expression if other parsers don't work - buf.position() = const_cast(token_iterator->begin); + buf->position() = const_cast(token_iterator->begin); std::pair value_raw = evaluateConstantExpression(ast, context); @@ -416,10 +428,10 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx type.insertDefaultInto(column); return false; } - buf.rollbackToCheckpoint(); + buf->rollbackToCheckpoint(); throw Exception{"Cannot insert NULL value into a column of type '" + type.getName() + "'" + " at: " + - String(buf.position(), std::min(SHOW_CHARS_ON_SYNTAX_ERROR, buf.buffer().end() - buf.position())), + String(buf->position(), std::min(SHOW_CHARS_ON_SYNTAX_ERROR, buf->buffer().end() - buf->position())), ErrorCodes::TYPE_MISMATCH}; } @@ -430,61 +442,61 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx /// Can be used in fileSegmentationEngine for parallel parsing of Values bool ValuesBlockInputFormat::skipToNextRow(size_t min_chunk_bytes, int balance) { - skipWhitespaceIfAny(buf); - if (buf.eof() || *buf.position() == ';') + skipWhitespaceIfAny(*buf); + if (buf->eof() || *buf->position() == ';') return false; bool quoted = false; - size_t chunk_begin_buf_count = buf.count(); - while (!buf.eof() && (balance || buf.count() - chunk_begin_buf_count < min_chunk_bytes)) + size_t chunk_begin_buf_count = buf->count(); + while (!buf->eof() && (balance || buf->count() - chunk_begin_buf_count < min_chunk_bytes)) { - buf.position() = find_first_symbols<'\\', '\'', ')', '('>(buf.position(), buf.buffer().end()); - if (buf.position() == buf.buffer().end()) + buf->position() = find_first_symbols<'\\', '\'', ')', '('>(buf->position(), buf->buffer().end()); + if (buf->position() == buf->buffer().end()) continue; - if (*buf.position() == '\\') + if (*buf->position() == '\\') { - ++buf.position(); - if (!buf.eof()) - ++buf.position(); + ++buf->position(); + if (!buf->eof()) + ++buf->position(); } - else if (*buf.position() == '\'') + else if (*buf->position() == '\'') { quoted ^= true; - ++buf.position(); + ++buf->position(); } - else if (*buf.position() == ')') + else if (*buf->position() == ')') { - ++buf.position(); + ++buf->position(); if (!quoted) --balance; } - else if (*buf.position() == '(') + else if (*buf->position() == '(') { - ++buf.position(); + ++buf->position(); if (!quoted) ++balance; } } - if (!buf.eof() && *buf.position() == ',') - ++buf.position(); + if (!buf->eof() && *buf->position() == ',') + ++buf->position(); return true; } void ValuesBlockInputFormat::assertDelimiterAfterValue(size_t column_idx) { if (unlikely(!checkDelimiterAfterValue(column_idx))) - throwAtAssertionFailed((column_idx + 1 == num_columns) ? ")" : ",", buf); + throwAtAssertionFailed((column_idx + 1 == num_columns) ? ")" : ",", *buf); } bool ValuesBlockInputFormat::checkDelimiterAfterValue(size_t column_idx) { - skipWhitespaceIfAny(buf); + skipWhitespaceIfAny(*buf); if (likely(column_idx + 1 != num_columns)) - return checkChar(',', buf); + return checkChar(',', *buf); else - return checkChar(')', buf); + return checkChar(')', *buf); } bool ValuesBlockInputFormat::shouldDeduceNewTemplate(size_t column_idx) @@ -516,21 +528,21 @@ bool ValuesBlockInputFormat::shouldDeduceNewTemplate(size_t column_idx) void ValuesBlockInputFormat::readPrefix() { /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it. - skipBOMIfExists(buf); + skipBOMIfExists(*buf); } void ValuesBlockInputFormat::readSuffix() { - if (!buf.eof() && *buf.position() == ';') + if (!buf->eof() && *buf->position() == ';') { - ++buf.position(); - skipWhitespaceIfAny(buf); - if (buf.hasUnreadData()) + ++buf->position(); + skipWhitespaceIfAny(*buf); + if (buf->hasUnreadData()) throw Exception("Cannot read data after semicolon", ErrorCodes::CANNOT_READ_ALL_DATA); return; } - if (buf.hasUnreadData()) + if (buf->hasUnreadData()) throw Exception("Unread data in PeekableReadBuffer will be lost. Most likely it's a bug.", ErrorCodes::LOGICAL_ERROR); } @@ -539,10 +551,16 @@ void ValuesBlockInputFormat::resetParser() IInputFormat::resetParser(); // I'm not resetting parser modes here. // There is a good chance that all messages have the same format. - buf.reset(); + buf->reset(); total_rows = 0; } +void ValuesBlockInputFormat::setReadBuffer(ReadBuffer & in_) +{ + buf = std::make_unique(in_); + IInputFormat::setReadBuffer(*buf); +} + void registerInputFormatValues(FormatFactory & factory) { factory.registerInputFormat("Values", []( diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.h b/src/Processors/Formats/Impl/ValuesBlockInputFormat.h index b1e44f5d104..5bbd4bea5ba 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.h @@ -32,6 +32,7 @@ public: String getName() const override { return "ValuesBlockInputFormat"; } void resetParser() override; + void setReadBuffer(ReadBuffer & in_) override; /// TODO: remove context somehow. void setContext(ContextPtr context_) { context = Context::createCopy(context_); } @@ -39,6 +40,9 @@ public: const BlockMissingValues & getMissingValues() const override { return block_missing_values; } private: + ValuesBlockInputFormat(std::unique_ptr buf_, const Block & header_, const RowInputFormatParams & params_, + const FormatSettings & format_settings_); + enum class ParserType { Streaming, @@ -66,7 +70,7 @@ private: bool skipToNextRow(size_t min_chunk_bytes = 0, int balance = 0); - PeekableReadBuffer buf; + std::unique_ptr buf; const RowInputFormatParams params; diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index 86998614189..64c8a01bb9c 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -42,12 +42,12 @@ InputFormatPtr getInputFormatFromASTInsertQuery( if (ast_insert_query->infile && context->getApplicationType() == Context::ApplicationType::SERVER) throw Exception("Query has infile and was send directly to server", ErrorCodes::UNKNOWN_TYPE_OF_QUERY); - String format = ast_insert_query->format; - if (format.empty()) + if (ast_insert_query->format.empty()) { if (input_function) throw Exception("FORMAT must be specified for function input()", ErrorCodes::INVALID_USAGE_OF_INPUT); - format = "Values"; + else + throw Exception("Logical error: INSERT query requires format to be set", ErrorCodes::LOGICAL_ERROR); } /// Data could be in parsed (ast_insert_query.data) and in not parsed yet (input_buffer_tail_part) part of query. @@ -59,7 +59,7 @@ InputFormatPtr getInputFormatFromASTInsertQuery( : std::make_unique(); /// Create a source from input buffer using format from query - auto source = context->getInputFormat(format, *input_buffer, header, context->getSettings().max_insert_block_size); + auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettings().max_insert_block_size); source->addBuffer(std::move(input_buffer)); return source; } diff --git a/tests/queries/0_stateless/02015_async_insert_7.reference b/tests/queries/0_stateless/02015_async_insert_7.reference new file mode 100644 index 00000000000..8816e320ee8 --- /dev/null +++ b/tests/queries/0_stateless/02015_async_insert_7.reference @@ -0,0 +1,6 @@ +1 a +2 b +3 c +4 d +5 e +6 f diff --git a/tests/queries/0_stateless/02015_async_insert_7.sh b/tests/queries/0_stateless/02015_async_insert_7.sh new file mode 100755 index 00000000000..c8cbbc48a29 --- /dev/null +++ b/tests/queries/0_stateless/02015_async_insert_7.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=1" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE async_inserts (id UInt32, s String) ENGINE = Memory" + +${CLICKHOUSE_CURL} -sS $url -d "INSERT INTO async_inserts VALUES (1, 'a') (2, 'b')" & +${CLICKHOUSE_CURL} -sS $url -d "INSERT INTO async_inserts VALUES (3, 'c'), (4, 'd')" & +${CLICKHOUSE_CURL} -sS $url -d "INSERT INTO async_inserts VALUES (5, 'e'), (6, 'f'), " & + +wait + +${CLICKHOUSE_CLIENT} -q "SELECT * FROM async_inserts ORDER BY id" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE async_inserts" From 1241db37950503f9eb72b72cf2172ae17fc8b620 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Wed, 20 Oct 2021 20:18:14 +0300 Subject: [PATCH 023/173] document the SQL-statements INTERSECT, EXCEPT and ANY, ALL operators --- docs/en/sql-reference/operators/index.md | 48 ++++++++++ .../sql-reference/statements/select/except.md | 86 ++++++++++++++++++ .../sql-reference/statements/select/index.md | 2 + .../statements/select/intersect.md | 87 +++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 docs/en/sql-reference/statements/select/except.md create mode 100644 docs/en/sql-reference/statements/select/intersect.md diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index dbda5f5dd69..159fc692e94 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -71,6 +71,54 @@ ClickHouse transforms operators to their corresponding functions at the query pa `a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` function. +`ALL` – The operator allows you to get data by comparing the value with the list of values returned by the subquery. The comparison condition must be running for all values of the subquery. + +The subquery must select values of the same type as those compared in the main predicate. + +**Example** + +Query: + +``` sql +SELECT number AS a FROM numbers(10) WHERE a > all(SELECT number FROM numbers(3, 3)); +``` + +Result: + +``` text +┌─a─┐ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└───┘ +``` + +`ANY` – The operator is comparing a value to a set of values returned by a subquery. The comparison condition must be running for any values of the subquery. + +The subquery must select values of the same type as those compared in the main predicate. + +**Example** + +Query: + +``` sql +SELECT number AS a FROM numbers(10) WHERE a > any(SELECT number FROM numbers(3, 3)); +``` + +Result: + +``` text +┌─a─┐ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└───┘ +``` + ## Operators for Working with Dates and Times {#operators-datetime} ### EXTRACT {#operator-extract} diff --git a/docs/en/sql-reference/statements/select/except.md b/docs/en/sql-reference/statements/select/except.md new file mode 100644 index 00000000000..ffbc0fae3da --- /dev/null +++ b/docs/en/sql-reference/statements/select/except.md @@ -0,0 +1,86 @@ +--- +toc_title: EXCEPT +--- + +# EXCEPT Clause {#except-clause} + +The `EXCEPT` clause returns only those rows that result from the first query without the second. The queries must match the number of columns, order, and type. The result of `EXCEPT` can contain duplicate rows. + +Multiple `EXCEPT` statements execute left to right if brackets are not specified. The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority as the `INTERSECT` clause. + +``` sql +SELECT column1 [, column2 ] +FROM table1 +[WHERE condition] + +EXCEPT + +SELECT column1 [, column2 ] +FROM table2 +[WHERE condition] + +``` +The condition could be any expression based on your requirements. + +**Example** + +Query: + +``` sql +SELECT number FROM numbers(1,10) EXCEPT SELECT number FROM numbers(3,6); +``` + +Result: + +``` text +┌─number─┐ +│ 1 │ +│ 2 │ +│ 9 │ +│ 10 │ +└────────┘ +``` + +Query: + +``` sql +SELECT number FROM numbers(1,10) WHERE number > 5 EXCEPT SELECT number FROM numbers(3,6) WHERE number < 7; +``` + +Result: + +``` text +┌─number─┐ +│ 7 │ +│ 8 │ +│ 9 │ +│ 10 │ +└────────┘ +``` + +Query: + +``` sql +CREATE TABLE t1(one String, two String, three String) ENGINE=Memory(); +CREATE TABLE t2(four String, five String, six String) ENGINE=Memory(); + +INSERT INTO t1 VALUES ('q', 'm', 'b'), ('s', 'd', 'f'), ('l', 'p', 'o'), ('s', 'd', 'f'), ('s', 'd', 'f'), ('k', 't', 'd'), ('l', 'p', 'o'); +INSERT INTO t2 VALUES ('q', 'm', 'b'), ('b', 'd', 'k'), ('s', 'y', 't'), ('s', 'd', 'f'), ('m', 'f', 'o'), ('k', 'k', 'd'); + +SELECT * FROM t1 EXCEPT SELECT * FROM t2; +``` + +Result: + +``` text +┌─one─┬─two─┬─three─┐ +│ l │ p │ o │ +│ k │ t │ d │ +│ l │ p │ o │ +└─────┴─────┴───────┘ +``` + +**See Also** + +- [UNION](union.md#union-clause) +- [INTERSECT](except.md#except-clause) \ No newline at end of file diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 0427764475a..39d12f880e4 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -49,6 +49,8 @@ Specifics of each optional clause are covered in separate sections, which are li - [LIMIT clause](../../../sql-reference/statements/select/limit.md) - [OFFSET clause](../../../sql-reference/statements/select/offset.md) - [UNION clause](../../../sql-reference/statements/select/union.md) +- [INTERSECT clause](../../../sql-reference/statements/select/intersect.md) +- [EXCEPT clause](../../../sql-reference/statements/select/except.md) - [INTO OUTFILE clause](../../../sql-reference/statements/select/into-outfile.md) - [FORMAT clause](../../../sql-reference/statements/select/format.md) diff --git a/docs/en/sql-reference/statements/select/intersect.md b/docs/en/sql-reference/statements/select/intersect.md new file mode 100644 index 00000000000..89446c68e62 --- /dev/null +++ b/docs/en/sql-reference/statements/select/intersect.md @@ -0,0 +1,87 @@ +--- +toc_title: INTERSECT +--- + +# INTERSECT Clause {#intersect-clause} + +The `INTERSECT` clause returns only those rows that result from first and second queries. The queries must match the number of columns, order, type, and names. The result of `INTERSECT` can contain duplicate rows. + +Multiple `INTERSECT` statements execute left to right if brackets are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause. + + +``` sql +SELECT column1 [, column2 ] +FROM table1 +[WHERE condition] + +INTERSECT + +SELECT column1 [, column2 ] +FROM table2 +[WHERE condition] + +``` +The condition could be any expression based on your requirements. + +**Example** + +Query: + +``` sql +SELECT number FROM numbers(1,10) INTERSECT SELECT number FROM numbers(3,6); +``` + +Result: + +``` text +┌─number─┐ +│ 3 │ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +└────────┘ +``` + +Query: + +``` sql +SELECT number FROM numbers(1,10) WHERE number > 5 INTERSECT SELECT number FROM numbers(3,6) WHERE number < 7; +``` + +Result: + +``` text +┌─number─┐ +│ 6 │ +└────────┘ +``` + +Query: + +``` sql +CREATE TABLE t1(one String, two String, three String) ENGINE=Memory(); +CREATE TABLE t2(four String, five String, six String) ENGINE=Memory(); + +INSERT INTO t1 VALUES ('q', 'm', 'b'), ('s', 'd', 'f'), ('l', 'p', 'o'), ('s', 'd', 'f'), ('s', 'd', 'f'), ('k', 't', 'd'), ('l', 'p', 'o'); +INSERT INTO t2 VALUES ('q', 'm', 'b'), ('b', 'd', 'k'), ('s', 'y', 't'), ('s', 'd', 'f'), ('m', 'f', 'o'), ('k', 'k', 'd'); + +SELECT * FROM t1 INTERSECT SELECT * FROM t2; +``` + +Result: + +``` text +┌─one─┬─two─┬─three─┐ +│ q │ m │ b │ +│ s │ d │ f │ +│ s │ d │ f │ +│ s │ d │ f │ +└─────┴─────┴───────┘ +``` + +**See Also** + +- [UNION](union.md#union-clause) +- [EXCEPT](except.md#except-clause) \ No newline at end of file From e36f06887bcccc1bb2bfc29844415b279987da21 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Wed, 20 Oct 2021 20:43:07 +0300 Subject: [PATCH 024/173] update --- docs/en/sql-reference/operators/index.md | 2 +- docs/en/sql-reference/statements/select/intersect.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 159fc692e94..7cc7a48cd77 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -94,7 +94,7 @@ Result: └───┘ ``` -`ANY` – The operator is comparing a value to a set of values returned by a subquery. The comparison condition must be running for any values of the subquery. +`ANY` – The operator is comparing a value to a set of values returned by a subquery. The comparison condition can be running for any values of the subquery. The subquery must select values of the same type as those compared in the main predicate. diff --git a/docs/en/sql-reference/statements/select/intersect.md b/docs/en/sql-reference/statements/select/intersect.md index 89446c68e62..95af5eb50a0 100644 --- a/docs/en/sql-reference/statements/select/intersect.md +++ b/docs/en/sql-reference/statements/select/intersect.md @@ -4,7 +4,7 @@ toc_title: INTERSECT # INTERSECT Clause {#intersect-clause} -The `INTERSECT` clause returns only those rows that result from first and second queries. The queries must match the number of columns, order, type, and names. The result of `INTERSECT` can contain duplicate rows. +The `INTERSECT` clause returns only those rows that result from first and second queries. The queries must match the number of columns, order, and type. The result of `INTERSECT` can contain duplicate rows. Multiple `INTERSECT` statements execute left to right if brackets are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause. From 81d9bac04a35edd0338869a5c5287818f9c5bc5c Mon Sep 17 00:00:00 2001 From: gyuton Date: Fri, 22 Oct 2021 03:53:44 +0300 Subject: [PATCH 025/173] Initial draft --- docs/en/operations/settings/settings.md | 77 +++++++++++++++++++ .../sql-reference/statements/insert-into.md | 1 + 2 files changed, 78 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2b088bf45bf..c0aa94d60b2 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3814,3 +3814,80 @@ Default value: `0`. **See Also** - [optimize_move_to_prewhere](#optimize_move_to_prewhere) setting + +## async_insert {#async-insert} + +Enables or disables asynchronous inserts for multiple concurrent connetions. This makes sense only for insertion over HTTP protocol. Note, that deduplications isn't working for such inserts. + +If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables. + +The data is inserted either after the [async-insert-max-data-size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout](#async_insert_busy_timeout) milliseconds since the first `INSERT` query. If the [async-insert-stale-timeout](#async-insert-stale-timeout) is set to a non-zero value, the data is inserted after `async-insert-stale-timeout` milliseconds since the last query. + +If [wait-for-async-insert](#wait-for-async-insert) is enabled, every client will wait for the data to be processed and flushed to the table. Otherwise, the query would be processed almost instantly, even if the data is not inserted. + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: `0`. + +## async_insert_threads {#async-insert-threads} + +The maximum number of threads for backgroung data parsing and insertion. + +Possible values: + +- Positive integer. +- 0 — Asynchronous insertions are disabled. + +Default value: `16`. + +## wait_for_async_insert {#wait-for-async-insert} + +Enables or disables waiting for processing of asynchronous insertion. If enabled, client will return `OK` only after the data is inserted. Otherwise, it will return `OK` even if the data wasn't inserted. + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: `1`. + +## wait_for_async_insert_timeout {#wait-for-async-insert-timeout} + +The timeout in seconds for waiting for processing of asynchronous insertion. + +- Positive integer. +- 0 — Disabled. + +Default value: [lock_acquire_timeout](#lock_acquire_timeout). + +## async_insert_max_data_size {#async-insert-max-data-size} + +The maximum size of the uparsed data in bytes collected per query before being inserted. + +Possible values: + +- Positive integer. +- 0 — Asynchronous insertions are disabled. + +Default value: `1000000`. + +## async_insert_busy_timeout {#async-insert-busy-timeout} + +The maximum timeout in milliseconds since the first `INSERT` query before inserting collected data. + +- Positive integer. +- 0 — Timeout disabled. + +Default value: `200`. + +## async_insert_stale_timeout {#async-insert-stale-timeout} + +The maximum timeout in milliseconds since the last `INSERT` query before dumping collected data. If enabled, the settings prolongs the [async_insert_busy_timeout](#async_insert_busy_timeout) with every `INSERT` query as long as [async-insert-max-data-size](#async-insert-max-data-size) is not exceeded. + +- Positive integer. +- 0 — Timeout disabled. + +Default value: `0`. diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 31b4d30835f..1bc69d9133f 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -119,3 +119,4 @@ Performance will not decrease if: - Data is added in real time. - You upload data that is usually sorted by time. +It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections with the setting [async-insert](../../operations/settings/settings.md#async-insert). The data from such inserts is combined into batches and then safely inserted into a table. From 8f039a2b0f35611e2ab8c1e3e5ebe9577e4fc550 Mon Sep 17 00:00:00 2001 From: gyuton Date: Fri, 22 Oct 2021 04:04:15 +0300 Subject: [PATCH 026/173] fixed links --- docs/en/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c0aa94d60b2..f02fe383f37 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3821,9 +3821,9 @@ Enables or disables asynchronous inserts for multiple concurrent connetions. Thi If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables. -The data is inserted either after the [async-insert-max-data-size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout](#async_insert_busy_timeout) milliseconds since the first `INSERT` query. If the [async-insert-stale-timeout](#async-insert-stale-timeout) is set to a non-zero value, the data is inserted after `async-insert-stale-timeout` milliseconds since the last query. +The data is inserted either after the [async_insert_max_data_size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout](#async-insert-busy-timeout) milliseconds since the first `INSERT` query. If the [async_insert_stale_timeout](#async-insert-stale-timeout) is set to a non-zero value, the data is inserted after `async_insert_stale_timeout` milliseconds since the last query. -If [wait-for-async-insert](#wait-for-async-insert) is enabled, every client will wait for the data to be processed and flushed to the table. Otherwise, the query would be processed almost instantly, even if the data is not inserted. +If [wait_for_async_insert](#wait-for-async-insert) is enabled, every client will wait for the data to be processed and flushed to the table. Otherwise, the query would be processed almost instantly, even if the data is not inserted. Possible values: @@ -3885,7 +3885,7 @@ Default value: `200`. ## async_insert_stale_timeout {#async-insert-stale-timeout} -The maximum timeout in milliseconds since the last `INSERT` query before dumping collected data. If enabled, the settings prolongs the [async_insert_busy_timeout](#async_insert_busy_timeout) with every `INSERT` query as long as [async-insert-max-data-size](#async-insert-max-data-size) is not exceeded. +The maximum timeout in milliseconds since the last `INSERT` query before dumping collected data. If enabled, the settings prolongs the [async_insert_busy_timeout](#async-insert-busy-timeout) with every `INSERT` query as long as [async_insert_max_data_size](#async-insert-max-data-size) is not exceeded. - Positive integer. - 0 — Timeout disabled. From 4ecd2ed6e204d4d70200810c43d5370dcbd61ee9 Mon Sep 17 00:00:00 2001 From: gyuton Date: Fri, 22 Oct 2021 04:21:29 +0300 Subject: [PATCH 027/173] typo --- docs/en/sql-reference/statements/insert-into.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 1bc69d9133f..83812a198d9 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -119,4 +119,4 @@ Performance will not decrease if: - Data is added in real time. - You upload data that is usually sorted by time. -It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections with the setting [async-insert](../../operations/settings/settings.md#async-insert). The data from such inserts is combined into batches and then safely inserted into a table. +It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections with the setting [async_insert](../../operations/settings/settings.md#async-insert). The data from such inserts is combined into batches and then safely inserted into a table. From 1e203cb604c956b5dcdbfbe01226930f880c8ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Fri, 22 Oct 2021 14:58:10 +0800 Subject: [PATCH 028/173] Retranslation --- .../system-tables/asynchronous_metric_log.md | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/zh/operations/system-tables/asynchronous_metric_log.md b/docs/zh/operations/system-tables/asynchronous_metric_log.md index ff7593768d3..6ae4a600539 100644 --- a/docs/zh/operations/system-tables/asynchronous_metric_log.md +++ b/docs/zh/operations/system-tables/asynchronous_metric_log.md @@ -5,4 +5,34 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ## system.asynchronous_metric_log {#system-tables-async-log} -包含以下内容的历史值 `system.asynchronous_log` （见 [系统。asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics)) +包含每分钟记录一次的 `system.asynchronous_metrics`历史值. 默认开启. + +列： +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件日期. +- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 事件时间. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件时间. +- `name` ([String](../../sql-reference/data-types/string.md)) — 指标名. +- `value` ([Float64](../../sql-reference/data-types/float.md)) — 指标值. + +**示例** +``` sql +SELECT * FROM system.asynchronous_metric_log LIMIT 10 +``` +``` text +┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0 │ 2120.9 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy │ 743 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty │ 26288 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │ 0 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs │ 0 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained │ 60694528 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped │ 303161344 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident │ 260931584 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata │ 12079488 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated │ 133756128 │ +└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘ +``` + +**另请参阅** +- [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics) — 包含在后台定期计算的指标. +- [system.metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log) — 包含定期刷新到磁盘表 `system.metrics` 以及 `system.events` 表中指标值的隶属记录. From bb8401546a31c3b19f792f783b0b109e31384b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Fri, 22 Oct 2021 15:04:17 +0800 Subject: [PATCH 029/173] retranslation --- .../system-tables/asynchronous_metric_log.md | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/zh/operations/system-tables/asynchronous_metric_log.md b/docs/zh/operations/system-tables/asynchronous_metric_log.md index ff7593768d3..6ae4a600539 100644 --- a/docs/zh/operations/system-tables/asynchronous_metric_log.md +++ b/docs/zh/operations/system-tables/asynchronous_metric_log.md @@ -5,4 +5,34 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ## system.asynchronous_metric_log {#system-tables-async-log} -包含以下内容的历史值 `system.asynchronous_log` （见 [系统。asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics)) +包含每分钟记录一次的 `system.asynchronous_metrics`历史值. 默认开启. + +列： +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件日期. +- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 事件时间. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件时间. +- `name` ([String](../../sql-reference/data-types/string.md)) — 指标名. +- `value` ([Float64](../../sql-reference/data-types/float.md)) — 指标值. + +**示例** +``` sql +SELECT * FROM system.asynchronous_metric_log LIMIT 10 +``` +``` text +┌─event_date─┬──────────event_time─┬────event_time_microseconds─┬─name─────────────────────────────────────┬─────value─┐ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ CPUFrequencyMHz_0 │ 2120.9 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pmuzzy │ 743 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.arenas.all.pdirty │ 26288 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.run_intervals │ 0 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.background_thread.num_runs │ 0 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.retained │ 60694528 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.mapped │ 303161344 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.resident │ 260931584 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.metadata │ 12079488 │ +│ 2020-09-05 │ 2020-09-05 15:56:30 │ 2020-09-05 15:56:30.025227 │ jemalloc.allocated │ 133756128 │ +└────────────┴─────────────────────┴────────────────────────────┴──────────────────────────────────────────┴───────────┘ +``` + +**另请参阅** +- [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics) — 包含在后台定期计算的指标. +- [system.metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log) — 包含定期刷新到磁盘表 `system.metrics` 以及 `system.events` 表中指标值的隶属记录. From 4ce26e173889ad1b1be0df43d1407f64dfd3ad98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Fri, 22 Oct 2021 15:59:17 +0800 Subject: [PATCH 030/173] Correct incorrect text --- docs/zh/operations/system-tables/asynchronous_metric_log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/operations/system-tables/asynchronous_metric_log.md b/docs/zh/operations/system-tables/asynchronous_metric_log.md index 6ae4a600539..60a3fd868a0 100644 --- a/docs/zh/operations/system-tables/asynchronous_metric_log.md +++ b/docs/zh/operations/system-tables/asynchronous_metric_log.md @@ -5,7 +5,7 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ## system.asynchronous_metric_log {#system-tables-async-log} -包含每分钟记录一次的 `system.asynchronous_metrics`历史值. 默认开启. +包含每分钟记录一次的 `system.asynchronous_metrics`表历史值. 默认开启. 列： - `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件日期. @@ -35,4 +35,4 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10 **另请参阅** - [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics) — 包含在后台定期计算的指标. -- [system.metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log) — 包含定期刷新到磁盘表 `system.metrics` 以及 `system.events` 表中指标值的隶属记录. +- [system.metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log) — 包含定期刷新到磁盘表 `system.metrics` 以及 `system.events` 中的指标值历史记录. From 44136e3da977c6e73348f84196e2f93a97ff8f86 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:46:44 +0300 Subject: [PATCH 031/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 7cc7a48cd77..134d67b0e37 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -71,7 +71,10 @@ ClickHouse transforms operators to their corresponding functions at the query pa `a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` function. -`ALL` – The operator allows you to get data by comparing the value with the list of values returned by the subquery. The comparison condition must be running for all values of the subquery. +`a = ANY (subquery)` – The `in(a, subquery)` function. +`a != ANY (subquery)` – The `notIn(a, subquery)` function. +`a = ALL (subquery)` – The same as `a IN (SELECT singleValueOrNull(*) FROM subquery)`. +`a != ALL (subquery)` – The same as `a NOT IN (subquery)`. The subquery must select values of the same type as those compared in the main predicate. From 0104af6c86d622b0afff604704bd66fa3d78adf3 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:46:56 +0300 Subject: [PATCH 032/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 134d67b0e37..a2d66481ddd 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -78,7 +78,7 @@ ClickHouse transforms operators to their corresponding functions at the query pa The subquery must select values of the same type as those compared in the main predicate. -**Example** +**Examples** Query: From 71d41421a08a3a517cfdc10a0f8af4c4edae5f3f Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:47:11 +0300 Subject: [PATCH 033/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index a2d66481ddd..a50c245de1b 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -76,7 +76,6 @@ ClickHouse transforms operators to their corresponding functions at the query pa `a = ALL (subquery)` – The same as `a IN (SELECT singleValueOrNull(*) FROM subquery)`. `a != ALL (subquery)` – The same as `a NOT IN (subquery)`. -The subquery must select values of the same type as those compared in the main predicate. **Examples** From 95cb825d214154dbf65ecb17a0d72cf5f8a3306c Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:47:22 +0300 Subject: [PATCH 034/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index a50c245de1b..3583d172939 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -79,7 +79,7 @@ ClickHouse transforms operators to their corresponding functions at the query pa **Examples** -Query: +Query with ALL: ``` sql SELECT number AS a FROM numbers(10) WHERE a > all(SELECT number FROM numbers(3, 3)); From 29618ff47d42a707975a303da6c321962f721b62 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:47:29 +0300 Subject: [PATCH 035/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 3583d172939..3370c2ee21e 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -82,7 +82,7 @@ ClickHouse transforms operators to their corresponding functions at the query pa Query with ALL: ``` sql -SELECT number AS a FROM numbers(10) WHERE a > all(SELECT number FROM numbers(3, 3)); +SELECT number AS a FROM numbers(10) WHERE a > ALL (SELECT number FROM numbers(3, 3)); ``` Result: From c7cc5a99f52a21f871269242cd686245b6ab0186 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:47:41 +0300 Subject: [PATCH 036/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 3370c2ee21e..4e3b539c731 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -102,7 +102,7 @@ The subquery must select values of the same type as those compared in the main p **Example** -Query: +Query with ANY: ``` sql SELECT number AS a FROM numbers(10) WHERE a > any(SELECT number FROM numbers(3, 3)); From bb7a179f9314f302e56ac9c58dfb92767e5c7072 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:47:48 +0300 Subject: [PATCH 037/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 4e3b539c731..f4aadababdb 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -105,7 +105,7 @@ The subquery must select values of the same type as those compared in the main p Query with ANY: ``` sql -SELECT number AS a FROM numbers(10) WHERE a > any(SELECT number FROM numbers(3, 3)); +SELECT number AS a FROM numbers(10) WHERE a > ANY (SELECT number FROM numbers(3, 3)); ``` Result: From cb907dd6c60d7e8f2d94d49792e6ae55c4c55b03 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:48:07 +0300 Subject: [PATCH 038/173] Update docs/en/sql-reference/statements/select/intersect.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/intersect.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/intersect.md b/docs/en/sql-reference/statements/select/intersect.md index 95af5eb50a0..58f8893a7a0 100644 --- a/docs/en/sql-reference/statements/select/intersect.md +++ b/docs/en/sql-reference/statements/select/intersect.md @@ -4,7 +4,7 @@ toc_title: INTERSECT # INTERSECT Clause {#intersect-clause} -The `INTERSECT` clause returns only those rows that result from first and second queries. The queries must match the number of columns, order, and type. The result of `INTERSECT` can contain duplicate rows. +The `INTERSECT` clause returns only those rows that result from both the first and the second queries. The queries must match the number of columns, order, and type. The result of `INTERSECT` can contain duplicate rows. Multiple `INTERSECT` statements execute left to right if brackets are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause. From 1b408ed588ff7ffa3e4aea8b5a311562af785544 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:48:31 +0300 Subject: [PATCH 039/173] Update docs/en/sql-reference/operators/index.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/operators/index.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index f4aadababdb..4ee22025674 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -96,12 +96,6 @@ Result: └───┘ ``` -`ANY` – The operator is comparing a value to a set of values returned by a subquery. The comparison condition can be running for any values of the subquery. - -The subquery must select values of the same type as those compared in the main predicate. - -**Example** - Query with ANY: ``` sql From 41f69e967712bb250e48dac2494888b03b01db54 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:48:53 +0300 Subject: [PATCH 040/173] Update docs/en/sql-reference/statements/select/except.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/except.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/except.md b/docs/en/sql-reference/statements/select/except.md index ffbc0fae3da..6702043e6b0 100644 --- a/docs/en/sql-reference/statements/select/except.md +++ b/docs/en/sql-reference/statements/select/except.md @@ -6,7 +6,7 @@ toc_title: EXCEPT The `EXCEPT` clause returns only those rows that result from the first query without the second. The queries must match the number of columns, order, and type. The result of `EXCEPT` can contain duplicate rows. -Multiple `EXCEPT` statements execute left to right if brackets are not specified. The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority as the `INTERSECT` clause. +Multiple `EXCEPT` statements are executed left to right if parenthesis are not specified. The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority than the `INTERSECT` clause. ``` sql SELECT column1 [, column2 ] From 692b99ca0d9e5e48cc7156fd68e12cf7576f421f Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:49:01 +0300 Subject: [PATCH 041/173] Update docs/en/sql-reference/statements/select/intersect.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/intersect.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/intersect.md b/docs/en/sql-reference/statements/select/intersect.md index 58f8893a7a0..cc75a2e2b4d 100644 --- a/docs/en/sql-reference/statements/select/intersect.md +++ b/docs/en/sql-reference/statements/select/intersect.md @@ -6,7 +6,7 @@ toc_title: INTERSECT The `INTERSECT` clause returns only those rows that result from both the first and the second queries. The queries must match the number of columns, order, and type. The result of `INTERSECT` can contain duplicate rows. -Multiple `INTERSECT` statements execute left to right if brackets are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause. +Multiple `INTERSECT` statements are executes left to right if parenthesis are not specified. The `INTERSECT` operator has a higher priority than the `UNION` and `EXCEPT` clause. ``` sql From 0f66ba1704a21423fbf5d5b60ed7557cd3ea1c2d Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:49:11 +0300 Subject: [PATCH 042/173] Update docs/en/sql-reference/statements/select/intersect.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/intersect.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/intersect.md b/docs/en/sql-reference/statements/select/intersect.md index cc75a2e2b4d..4a222d80d8d 100644 --- a/docs/en/sql-reference/statements/select/intersect.md +++ b/docs/en/sql-reference/statements/select/intersect.md @@ -23,7 +23,7 @@ FROM table2 ``` The condition could be any expression based on your requirements. -**Example** +**Examples** Query: From 745b47762cdd24cf6ec762ce28ced0e245514c60 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:49:20 +0300 Subject: [PATCH 043/173] Update docs/en/sql-reference/statements/select/except.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/select/except.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/select/except.md b/docs/en/sql-reference/statements/select/except.md index 6702043e6b0..362ddb29b0e 100644 --- a/docs/en/sql-reference/statements/select/except.md +++ b/docs/en/sql-reference/statements/select/except.md @@ -22,7 +22,7 @@ FROM table2 ``` The condition could be any expression based on your requirements. -**Example** +**Examples** Query: From ca6b6df93e7a7bc71d8f275ed935415ba0deaad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Fri, 22 Oct 2021 16:49:24 +0800 Subject: [PATCH 044/173] translation for Continuous Integration Checks Document: Chinese translation for Continuous Integration Checks --- docs/zh/development/continuous-integration.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index f68058a436e..8f147b68cf1 120000 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -1 +1,18 @@ -../../en/development/continuous-integration.md \ No newline at end of file +# 持续集成检查 {#continuous-integration-checks} +当你提交一个pull请求时, ClickHouse的[持续集成(CI)系统](../tests.md#test-automation)会对您的代码运行一些自动检查. + +这在存储库维护者(来自ClickHouse团队的人)筛选了您的代码并将可测试标签添加到您的pull请求之后发生. + +检查的结果被列在[GitHub检查文档](https://docs.github.com/en/github/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks)中描述的GitHub pull请求页面. + +如果检查失败，您可能被要求去修复它. 该界面介绍了您可能遇到的检查，以及如何修复它们. + +如果检查失败看起来与您的更改无关，那么它可能是一些暂时的故障或基础设施问题. 向pull请求推一个空的commit以重新启动CI检查: + +``` +git reset +git commit --allow-empty +git push +``` + +如果您不确定要做什么，可以向维护人员寻求帮助. From 3510f4cddcc8c07f4358cb5dc3c1c8829a95090e Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:58:04 +0300 Subject: [PATCH 045/173] Update except.md --- .../sql-reference/statements/select/except.md | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/docs/en/sql-reference/statements/select/except.md b/docs/en/sql-reference/statements/select/except.md index 362ddb29b0e..d45c0556a1a 100644 --- a/docs/en/sql-reference/statements/select/except.md +++ b/docs/en/sql-reference/statements/select/except.md @@ -43,23 +43,6 @@ Result: Query: -``` sql -SELECT number FROM numbers(1,10) WHERE number > 5 EXCEPT SELECT number FROM numbers(3,6) WHERE number < 7; -``` - -Result: - -``` text -┌─number─┐ -│ 7 │ -│ 8 │ -│ 9 │ -│ 10 │ -└────────┘ -``` - -Query: - ``` sql CREATE TABLE t1(one String, two String, three String) ENGINE=Memory(); CREATE TABLE t2(four String, five String, six String) ENGINE=Memory(); @@ -83,4 +66,4 @@ Result: **See Also** - [UNION](union.md#union-clause) -- [INTERSECT](except.md#except-clause) \ No newline at end of file +- [INTERSECT](except.md#except-clause) From e4df8417981ab0d6e106454965685cc89341c657 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 11:58:34 +0300 Subject: [PATCH 046/173] Update intersect.md --- .../sql-reference/statements/select/intersect.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/en/sql-reference/statements/select/intersect.md b/docs/en/sql-reference/statements/select/intersect.md index 4a222d80d8d..2243a35e4d8 100644 --- a/docs/en/sql-reference/statements/select/intersect.md +++ b/docs/en/sql-reference/statements/select/intersect.md @@ -46,20 +46,6 @@ Result: Query: -``` sql -SELECT number FROM numbers(1,10) WHERE number > 5 INTERSECT SELECT number FROM numbers(3,6) WHERE number < 7; -``` - -Result: - -``` text -┌─number─┐ -│ 6 │ -└────────┘ -``` - -Query: - ``` sql CREATE TABLE t1(one String, two String, three String) ENGINE=Memory(); CREATE TABLE t2(four String, five String, six String) ENGINE=Memory(); @@ -84,4 +70,4 @@ Result: **See Also** - [UNION](union.md#union-clause) -- [EXCEPT](except.md#except-clause) \ No newline at end of file +- [EXCEPT](except.md#except-clause) From fc3f577cbfba6809d51f8900ec1a14866c7fc00f Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 12:14:33 +0300 Subject: [PATCH 047/173] Update index.md --- docs/en/sql-reference/operators/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 4ee22025674..d23c81e2aca 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -72,8 +72,11 @@ ClickHouse transforms operators to their corresponding functions at the query pa `a GLOBAL NOT IN ...` – The `globalNotIn(a, b)` function. `a = ANY (subquery)` – The `in(a, subquery)` function. + `a != ANY (subquery)` – The `notIn(a, subquery)` function. + `a = ALL (subquery)` – The same as `a IN (SELECT singleValueOrNull(*) FROM subquery)`. + `a != ALL (subquery)` – The same as `a NOT IN (subquery)`. From 3c8b32dfe6f46973ee533a18db99d240f31e9e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Fri, 22 Oct 2021 17:20:10 +0800 Subject: [PATCH 048/173] =?UTF-8?q?=E7=AC=AC=E4=BA=8C=E6=AE=B5=E7=BF=BB?= =?UTF-8?q?=E8=AF=91=E5=AD=98=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/zh/development/continuous-integration.md | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 8f147b68cf1..70a88702a92 120000 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -7,7 +7,7 @@ 如果检查失败，您可能被要求去修复它. 该界面介绍了您可能遇到的检查，以及如何修复它们. -如果检查失败看起来与您的更改无关，那么它可能是一些暂时的故障或基础设施问题. 向pull请求推一个空的commit以重新启动CI检查: +如果检查失败看起来与您的更改无关, 那么它可能是一些暂时的故障或基础设施问题. 向pull请求推一个空的commit以重新启动CI检查: ``` git reset @@ -16,3 +16,36 @@ git push ``` 如果您不确定要做什么，可以向维护人员寻求帮助. + + +# 与Master合并 {#merge-with-master} +验证PR是否可以合并到master. 如果没有, 它将返回消息为'Cannot fetch mergecommit'的失败.要修复这个检查, 解决[GitHub文档](https://docs.github.com/en/github/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-on-github)中描述的冲突, 或者使用git合并主分支到你的pull request分支. + +# 文档检查 {#docs-check} +尝试构建ClickHouse文档网站. 如果您更改了文档中的某些内容, 它可能会失败. 最可能的原因是文档中的某些交叉链接是错误的. 转到检查报告并查找`ERROR`和`WARNING`消息. + +# 查看报告详情 {#report-details} +- [状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +- `docs_output.txt`记录了build日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) + +# 描述信息检查 {#description-check} +检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](https://github.com/ClickHouse/ClickHouse/blob/master/.github/PULL_REQUEST_TEMPLATE.md)模板. + +您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](https://clickhouse.com/docs/en/whats-new/changelog/)编写一条用户可读的消息用来描述更改. + +# 推送到DockerHub {#push-to-dockerhub} +生成用于构建和测试的docker映像, 然后将它们推送到DockerHub. + +# 标记检查 {#marker-check} +该检查意味着集成系统已经开始处理PR.当它处于待定状态时, 意味着还没有开始所有的检查. 在所有检查启动后，它将状态更改为'成功'. + +# 格式检查 {#style-check} +使用`utils/check-style/check-style`二进制文件执行一些简单的基于正则表达式的代码样式检查(注意, 它可以在本地运行). +如果失败, 按照[代码样式指南](https://clickhouse.com/docs/en/development/style/)修复样式错误. + +# 查看报告详情 {#report-details} +- [状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +- `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) + +# PVS Check {#pvs-check} +使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告找出确切的错误.如果可以则修复它们, 如果不行, 可以请ClickHouse的维护人员帮忙. From 40e35a4571b8617dda035e27f232459aaec00779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Fri, 22 Oct 2021 17:58:02 +0800 Subject: [PATCH 049/173] Archive translated documents --- docs/zh/development/continuous-integration.md | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 70a88702a92..205a7edf15e 120000 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -49,3 +49,68 @@ git push # PVS Check {#pvs-check} 使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告找出确切的错误.如果可以则修复它们, 如果不行, 可以请ClickHouse的维护人员帮忙. + +# 查看报告详情 {#report-details} +- [状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +- `test_run.txt.out.log`记录了构建和分析日志文件.它只包含解析或未找到的错误. +- `HTML report`记录了分析结果.关于它的描述, 请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) + +# {#fast-test} +通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 通过报告查看哪些测试失败, 然后按照[这里](https://clickhouse.com/docs/en/development/tests/#functional-test-locally)描述的在本地重现失败. + +# 查看报告详情 {#report-details} +[状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) + +# 状态页文件 {#status-page-files} +- `runlog.out.log` 是包含所有其他日志的通用日志. +- `test_log.txt` +- `submodule_log.txt` 包含关于克隆和检查所需子模块的消息. +- `stderr.log` +- `stdout.log` +- `clickhouse-server.log` +- `clone_log.txt` +- `install_log.txt` +- `clickhouse-server.err.log` +- `build_log.txt` +- `cmake_log.txt` 包含关于C/ c++和Linux标志检查的消息. + +# 状态页列信息 {#status-page-columns} +- 测试名称 -- 包含测试的名称(没有路径, 例如, 所有类型的测试将被剥离到该名称). +- 测试状态 -- 跳过、成功或失败. +- 测试时间, 等等. -- 这个测试是空的. + +# 建构检查 {#build-check} +在各种配置中构建ClickHouse, 以便在后续步骤中使用. 您必须修复失败的构建.构建日志通常有足够的信息来修复错误, 但是您可能必须在本地重新生成错误. cmake选项可以在构建日志中找到, 为cmake做准备.使用这些选项并遵循一般的构建过程. + +# 查看报告详情 {#report-details} +[状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +- **Compiler**: `gcc-9` or `clang-10` (or `clang-10-xx` for other architectures e.g. `clang-10-freebsd`). +- **Build type**: `Debug` or `RelWithDebInfo` (cmake). +- **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan). +- **Bundled**: `bundled` 构建使用来自 `contrib` 库, 而 `unbundled` 构建使用系统库. +- **Splitted**: `splitted` is a [split build](https://clickhouse.com/docs/en/development/build/#split-build) +- **Status**: `成功` 或 `失败` +- **Build log**: 链接到生成和文件复制日志，这在生成失败时很有用. +- **Build time**. +- **Artifacts**: 构建结果文件 (with `XXX` being the server version e.g. `20.8.1.4344`). + - `clickhouse-client_XXX_all.deb` + -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` + - `clickhouse-common-staticXXX_amd64.deb` + - `clickhouse-server_XXX_all.deb` + - `clickhouse-test_XXX_all.deb` + - `clickhouse_XXX_amd64.buildinfo` + - `clickhouse_XXX_amd64.changes` + - `clickhouse`: Main built binary. + - `clickhouse-odbc-bridge` + - `unit_tests_dbms`: GoogleTest二进制和ClickHouse单元测试. + - `shared_build.tgz`: 使用共享库构建. + - `performance.tgz`: 用于性能测试的特殊包. + +# 特殊的构建检查 {#special-buildcheck} +使用clang-tidy执行静态分析和代码风格检查. 报告类似于构建检查. 修复在构建日志中发现的错误. + + + + + + From add7c000e19c3c95df19556fd724aaa2d7dee084 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 22 Oct 2021 16:32:49 +0300 Subject: [PATCH 050/173] Update test_aggregate_fixed_key.py --- .../test_backward_compatibility/test_aggregate_fixed_key.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index d895ee0eb78..8819be527fd 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -3,9 +3,9 @@ import pytest from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', with_zookeeper=True, image='clickhouse/server', tag='21.3', with_installed_binary=True) -node2 = cluster.add_instance('node2', with_zookeeper=True, image='clickhouse/server') -node3 = cluster.add_instance('node3', with_zookeeper=True, image='clickhouse/server') +node1 = cluster.add_instance('node1', with_zookeeper=True, image='yandex/clickhouse-server', tag='21.3', with_installed_binary=True) +node2 = cluster.add_instance('node2', with_zookeeper=True, image='yandex/clickhouse-server') +node3 = cluster.add_instance('node3', with_zookeeper=True, image='yandex/clickhouse-server') @pytest.fixture(scope="module") From e00d55c08d416300810ccc52a77bbf95c4caab67 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 22 Oct 2021 17:42:20 +0300 Subject: [PATCH 051/173] Try to fix 30397 --- src/Storages/StorageBuffer.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index a74223054e9..1d1c3456813 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -817,8 +817,6 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc if (!locked) lock.emplace(buffer.lockForReading()); - block_to_write = buffer.data.cloneEmpty(); - rows = buffer.data.rows(); bytes = buffer.data.bytes(); if (buffer.first_write_time) From ee6bc3883fd2df7801be1f5de23a07cb2d889365 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 22 Oct 2021 19:25:48 +0300 Subject: [PATCH 052/173] Try to fix 30397 --- src/Storages/StorageBuffer.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 1d1c3456813..549f87c9f99 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -446,7 +446,8 @@ static void appendBlock(const Block & from, Block & to) if (!to) throw Exception("Cannot append to empty block", ErrorCodes::LOGICAL_ERROR); - assertBlocksHaveEqualStructure(from, to, "Buffer"); + if (to.rows()) + assertBlocksHaveEqualStructure(from, to, "Buffer"); from.checkNumberOfRows(); to.checkNumberOfRows(); @@ -464,14 +465,21 @@ static void appendBlock(const Block & from, Block & to) { MemoryTracker::BlockerInThread temporarily_disable_memory_tracker; - for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) + if (to.rows() == 0) { - const IColumn & col_from = *from.getByPosition(column_no).column.get(); - last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); + to = from; + } + else + { + for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) + { + const IColumn & col_from = *from.getByPosition(column_no).column.get(); + last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); - last_col->insertRangeFrom(col_from, 0, rows); + last_col->insertRangeFrom(col_from, 0, rows); - to.getByPosition(column_no).column = std::move(last_col); + to.getByPosition(column_no).column = std::move(last_col); + } } } catch (...) @@ -817,6 +825,8 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc if (!locked) lock.emplace(buffer.lockForReading()); + block_to_write = buffer.data.cloneEmpty(); + rows = buffer.data.rows(); bytes = buffer.data.bytes(); if (buffer.first_write_time) From ae692b96c6ac2d1c296f7507fc6a9b778dd458d7 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 21:57:38 +0300 Subject: [PATCH 053/173] russian --- docs/ru/sql-reference/operators/index.md | 47 ++++++++++++ .../sql-reference/statements/select/except.md | 69 ++++++++++++++++++ .../sql-reference/statements/select/index.md | 4 +- .../statements/select/intersect.md | 73 +++++++++++++++++++ 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 docs/ru/sql-reference/statements/select/except.md create mode 100644 docs/ru/sql-reference/statements/select/intersect.md diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 98f6873f712..19119f641ef 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -72,6 +72,53 @@ toc_title: "Операторы" `a GLOBAL NOT IN ...` - функция `globalNotIn(a, b)` +`a = ANY (subquery)` – функция `in(a, subquery)` + +`a != ANY (subquery)` – функция `notIn(a, subquery)` + +`a = ALL (subquery)` – равнозначно `a IN (SELECT singleValueOrNull(*) FROM subquery)` + +`a != ALL (subquery)` – равнозначно `a NOT IN (subquery)` + + +**Примеры** + +Запрос с ALL: + +``` sql +SELECT number AS a FROM numbers(10) WHERE a > ALL (SELECT number FROM numbers(3, 3)); +``` + +Результат: + +``` text +┌─a─┐ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└───┘ +``` + +Запрос с ANY: + +``` sql +SELECT number AS a FROM numbers(10) WHERE a > ANY (SELECT number FROM numbers(3, 3)); +``` + +Результат: + +``` text +┌─a─┐ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└───┘ +``` + ## Оператор для работы с датами и временем {#operators-datetime} ### EXTRACT {#extract} diff --git a/docs/ru/sql-reference/statements/select/except.md b/docs/ru/sql-reference/statements/select/except.md new file mode 100644 index 00000000000..6929eff1c81 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/except.md @@ -0,0 +1,69 @@ +--- +toc_title: EXCEPT +--- + +# Секция EXCEPT {#except-clause} + +Оператор `EXCEPT` возвращает только те строки, которые являются результатом первого запроса без результатов второго. В запросах количество, порядок следования и типы столбцов должны совпадать. Результат `EXCEPT` может содержать повторяющиеся строки. + +Если используется несколько `EXCEPT`, и в выражении не указаны скобки, `EXCEPT` выполняется по порядку слева направо. Оператор `EXCEPT` имеет такой же приоритет выполнения, как `UNION`, и ниже, чем `INTERSECT`. + +``` sql +SELECT column1 [, column2 ] +FROM table1 +[WHERE condition] + +EXCEPT + +SELECT column1 [, column2 ] +FROM table2 +[WHERE condition] + +``` +Условие в секции `WHERE` может быть любым в зависимости от ваших требований. + +**Примеры** + +Запрос: + +``` sql +SELECT number FROM numbers(1,10) EXCEPT SELECT number FROM numbers(3,6); +``` + +Результат: + +``` text +┌─number─┐ +│ 1 │ +│ 2 │ +│ 9 │ +│ 10 │ +└────────┘ +``` + +Запрос: + +``` sql +CREATE TABLE t1(one String, two String, three String) ENGINE=Memory(); +CREATE TABLE t2(four String, five String, six String) ENGINE=Memory(); + +INSERT INTO t1 VALUES ('q', 'm', 'b'), ('s', 'd', 'f'), ('l', 'p', 'o'), ('s', 'd', 'f'), ('s', 'd', 'f'), ('k', 't', 'd'), ('l', 'p', 'o'); +INSERT INTO t2 VALUES ('q', 'm', 'b'), ('b', 'd', 'k'), ('s', 'y', 't'), ('s', 'd', 'f'), ('m', 'f', 'o'), ('k', 'k', 'd'); + +SELECT * FROM t1 EXCEPT SELECT * FROM t2; +``` + +Результат: + +``` text +┌─one─┬─two─┬─three─┐ +│ l │ p │ o │ +│ k │ t │ d │ +│ l │ p │ o │ +└─────┴─────┴───────┘ +``` + +**См. также** + +- [UNION](union.md#union-clause) +- [INTERSECT](except.md#except-clause) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md index ffaae74b1d9..42490a6c2f7 100644 --- a/docs/ru/sql-reference/statements/select/index.md +++ b/docs/ru/sql-reference/statements/select/index.md @@ -45,8 +45,10 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list - [Секция LIMIT BY](limit-by.md) - [Секция HAVING](having.md) - [Секция LIMIT](limit.md) - [Секция OFFSET](offset.md) +- [Секция OFFSET](offset.md) - [Секция UNION ALL](union.md) +- [Секция INTERSECT](intersect.md) +- [Секция EXCEPT](except.md) - [Секция INTO OUTFILE](into-outfile.md) - [Секция FORMAT](format.md) diff --git a/docs/ru/sql-reference/statements/select/intersect.md b/docs/ru/sql-reference/statements/select/intersect.md new file mode 100644 index 00000000000..def2ff8b9f9 --- /dev/null +++ b/docs/ru/sql-reference/statements/select/intersect.md @@ -0,0 +1,73 @@ +--- +toc_title: INTERSECT +--- + +# INTERSECT Clause {#intersect-clause} + +Оператор `INTERSECT` возвращает только те строки, которые есть только в результатах первого и второго запросов. В запросах должны совпадать количество столбцов, их порядок и тип. Результат `INTERSECT` может содержать повторяющиеся строки. + +Если используется несколько операторов `INTERSECT` и скобки не указаны, пересечение выполняется слева направо. У оператора `INTERSECT` более высокий приоритет выполнения, чем у `UNION` и `EXCEPT`. + + +``` sql +SELECT column1 [, column2 ] +FROM table1 +[WHERE condition] + +INTERSECT + +SELECT column1 [, column2 ] +FROM table2 +[WHERE condition] + +``` +Условие может быть любым в зависимости от ваших требований. + +**Примеры** + +Запрос: + +``` sql +SELECT number FROM numbers(1,10) INTERSECT SELECT number FROM numbers(3,6); +``` + +Результат: + +``` text +┌─number─┐ +│ 3 │ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +└────────┘ +``` + +Запрос: + +``` sql +CREATE TABLE t1(one String, two String, three String) ENGINE=Memory(); +CREATE TABLE t2(four String, five String, six String) ENGINE=Memory(); + +INSERT INTO t1 VALUES ('q', 'm', 'b'), ('s', 'd', 'f'), ('l', 'p', 'o'), ('s', 'd', 'f'), ('s', 'd', 'f'), ('k', 't', 'd'), ('l', 'p', 'o'); +INSERT INTO t2 VALUES ('q', 'm', 'b'), ('b', 'd', 'k'), ('s', 'y', 't'), ('s', 'd', 'f'), ('m', 'f', 'o'), ('k', 'k', 'd'); + +SELECT * FROM t1 INTERSECT SELECT * FROM t2; +``` + +Результат: + +``` text +┌─one─┬─two─┬─three─┐ +│ q │ m │ b │ +│ s │ d │ f │ +│ s │ d │ f │ +│ s │ d │ f │ +└─────┴─────┴───────┘ +``` + +**См. также** + +- [UNION](union.md#union-clause) +- [EXCEPT](except.md#except-clause) From 82fc693d55871467439b888db0d19463e87025be Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Fri, 22 Oct 2021 22:38:34 +0300 Subject: [PATCH 054/173] russian fixes --- docs/en/sql-reference/statements/select/except.md | 2 +- docs/ru/sql-reference/statements/select/except.md | 6 +++--- docs/ru/sql-reference/statements/select/intersect.md | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/statements/select/except.md b/docs/en/sql-reference/statements/select/except.md index d45c0556a1a..e6d9b365a91 100644 --- a/docs/en/sql-reference/statements/select/except.md +++ b/docs/en/sql-reference/statements/select/except.md @@ -66,4 +66,4 @@ Result: **See Also** - [UNION](union.md#union-clause) -- [INTERSECT](except.md#except-clause) +- [INTERSECT](intersect.md#intersect-clause) diff --git a/docs/ru/sql-reference/statements/select/except.md b/docs/ru/sql-reference/statements/select/except.md index 6929eff1c81..eb4986aceea 100644 --- a/docs/ru/sql-reference/statements/select/except.md +++ b/docs/ru/sql-reference/statements/select/except.md @@ -4,9 +4,9 @@ toc_title: EXCEPT # Секция EXCEPT {#except-clause} -Оператор `EXCEPT` возвращает только те строки, которые являются результатом первого запроса без результатов второго. В запросах количество, порядок следования и типы столбцов должны совпадать. Результат `EXCEPT` может содержать повторяющиеся строки. +`EXCEPT` возвращает только те строки, которые являются результатом первого запроса без результатов второго. В запросах количество, порядок следования и типы столбцов должны совпадать. Результат `EXCEPT` может содержать повторяющиеся строки. -Если используется несколько `EXCEPT`, и в выражении не указаны скобки, `EXCEPT` выполняется по порядку слева направо. Оператор `EXCEPT` имеет такой же приоритет выполнения, как `UNION`, и ниже, чем `INTERSECT`. +Если используется несколько `EXCEPT`, и в выражении не указаны скобки, `EXCEPT` выполняется по порядку слева направо. `EXCEPT` имеет такой же приоритет выполнения, как `UNION`, и приоритет ниже, чем у `INTERSECT`. ``` sql SELECT column1 [, column2 ] @@ -66,4 +66,4 @@ SELECT * FROM t1 EXCEPT SELECT * FROM t2; **См. также** - [UNION](union.md#union-clause) -- [INTERSECT](except.md#except-clause) \ No newline at end of file +- [INTERSECT](intersect.md#intersect-clause) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/select/intersect.md b/docs/ru/sql-reference/statements/select/intersect.md index def2ff8b9f9..cf0c5aee55a 100644 --- a/docs/ru/sql-reference/statements/select/intersect.md +++ b/docs/ru/sql-reference/statements/select/intersect.md @@ -2,11 +2,11 @@ toc_title: INTERSECT --- -# INTERSECT Clause {#intersect-clause} +# Секция INTERSECT {#intersect-clause} -Оператор `INTERSECT` возвращает только те строки, которые есть только в результатах первого и второго запросов. В запросах должны совпадать количество столбцов, их порядок и тип. Результат `INTERSECT` может содержать повторяющиеся строки. +`INTERSECT` возвращает только те строки, которые есть только в результатах первого и второго запросов. В запросах должны совпадать количество столбцов, их порядок и тип. Результат `INTERSECT` может содержать повторяющиеся строки. -Если используется несколько операторов `INTERSECT` и скобки не указаны, пересечение выполняется слева направо. У оператора `INTERSECT` более высокий приоритет выполнения, чем у `UNION` и `EXCEPT`. +Если используется несколько `INTERSECT` и скобки не указаны, пересечение выполняется слева направо. У `INTERSECT` более высокий приоритет выполнения, чем у `UNION` и `EXCEPT`. ``` sql From 258f8cb2f7ccd22554c8c050fe265120284eeac4 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 22 Oct 2021 23:10:02 +0300 Subject: [PATCH 055/173] Add normalizeUTF8 functions. --- .../functions/string-functions.md | 126 +++++++++++++++++- 1 file changed, 125 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 4ac3a175aba..f6dbadb875a 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -826,11 +826,135 @@ normalizeUTF8NFKD(words) **Returned value** -- String transformed to normalization form C (NFC). +- String transformed to NFC normalization form. Type: [String](../../sql-reference/data-types/string.md). +**Example** +Query: + +``` sql +SELECT length('ё'), normalizeUTF8NFC('ё') AS nfc, length(nfc) AS nfc_len; +``` + +Result: + +``` text +┌─length('ё')─┬─nfc─┬─nfc_len─┐ +│ 2 │ ё │ 2 │ +└─────────────┴─────┴─────────┘ +``` + +## normalizeUTF8NFD {#normalizeutf8nfd} + +Converts a string to [NFD normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text. + +**Syntax** + +``` sql +normalizeUTF8NFD(words) +``` + +**Arguments** + +- `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- String transformed to NFD normalization form. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT length('ё'), normalizeUTF8NFD('ё') AS nfd, length(nfd) AS nfd_len; +``` + +Result: + +``` text +┌─length('ё')─┬─nfd─┬─nfd_len─┐ +│ 2 │ ё │ 4 │ +└─────────────┴─────┴─────────┘ +``` + +## normalizeUTF8NFKC {#normalizeutf8nfkc} + +Converts a string to [NFKC normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text. + +**Syntax** + +``` sql +normalizeUTF8NFKC(words) +``` + +**Arguments** + +- `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- String transformed to NFKC normalization form. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT length('ё'), normalizeUTF8NFKC('ё') AS nfkc, length(nfkc) AS nfkc_len; +``` + +Result: + +``` text +┌─length('ё')─┬─nfkc─┬─nfkc_len─┐ +│ 2 │ ё │ 2 │ +└─────────────┴──────┴──────────┘ + +``` + +## normalizeUTF8NFKD {#normalizeutf8nfkd} + + +Converts a string to [NFKD normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text. + +**Syntax** + +``` sql +normalizeUTF8NFKD(words) +``` + +**Arguments** + +- `words` — Input string that contains UTF-8 encoded text. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- String transformed to NFKD normalization form. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +Query: + +``` sql +SELECT length('ё'), normalizeUTF8NFKD('ё') AS nfkd, length(nfkd) AS nfkd_len; +``` + +Result: + +``` text +┌─length('ё')─┬─nfkd─┬─nfkd_len─┐ +│ 2 │ ё │ 4 │ +└─────────────┴──────┴──────────┘ +``` ## encodeXMLComponent {#encode-xml-component} From 99800cc8d4f529c09ba25d3708645b08613e4990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Sat, 23 Oct 2021 11:28:22 +0800 Subject: [PATCH 056/173] Translation for Continuous Integration Checks --- docs/zh/development/continuous-integration.md | 113 ++++++++++++------ 1 file changed, 76 insertions(+), 37 deletions(-) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 205a7edf15e..7e4c1bf36a9 120000 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -1,9 +1,9 @@ # 持续集成检查 {#continuous-integration-checks} -当你提交一个pull请求时, ClickHouse的[持续集成(CI)系统](../tests.md#test-automation)会对您的代码运行一些自动检查. +当你提交一个pull请求时, ClickHouse[持续集成(CI)系统](../tests.md#test-automation)会对您的代码运行一些自动检查. 这在存储库维护者(来自ClickHouse团队的人)筛选了您的代码并将可测试标签添加到您的pull请求之后发生. -检查的结果被列在[GitHub检查文档](https://docs.github.com/en/github/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks)中描述的GitHub pull请求页面. +检查的结果被列在[GitHub检查文档](https://docs.github.com/en/github/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks)中所述的GitHub pull请求页面. 如果检查失败，您可能被要求去修复它. 该界面介绍了您可能遇到的检查，以及如何修复它们. @@ -17,49 +17,48 @@ git push 如果您不确定要做什么，可以向维护人员寻求帮助. - # 与Master合并 {#merge-with-master} -验证PR是否可以合并到master. 如果没有, 它将返回消息为'Cannot fetch mergecommit'的失败.要修复这个检查, 解决[GitHub文档](https://docs.github.com/en/github/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-on-github)中描述的冲突, 或者使用git合并主分支到你的pull request分支. +验证PR是否可以合并到master. 如果没有, 它将失败并显示消息'Cannot fetch mergecommit'的.请按[GitHub文档](https://docs.github.com/en/github/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-on-github)中描述的冲突解决, 或使用git将主分支合并到您的pull请求分支来修复这个检查. # 文档检查 {#docs-check} 尝试构建ClickHouse文档网站. 如果您更改了文档中的某些内容, 它可能会失败. 最可能的原因是文档中的某些交叉链接是错误的. 转到检查报告并查找`ERROR`和`WARNING`消息. -# 查看报告详情 {#report-details} -- [状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) -- `docs_output.txt`记录了build日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) +# 报告详情 {#report-details} +- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +- `docs_output.txt`包含构建日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) # 描述信息检查 {#description-check} -检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](https://github.com/ClickHouse/ClickHouse/blob/master/.github/PULL_REQUEST_TEMPLATE.md)模板. +检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](../../../.github/PULL_REQUEST_TEMPLATE.md)模板. -您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](https://clickhouse.com/docs/en/whats-new/changelog/)编写一条用户可读的消息用来描述更改. +您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](../whats-new/changelog/)编写一条用户可读的消息用来描述更改. # 推送到DockerHub {#push-to-dockerhub} 生成用于构建和测试的docker映像, 然后将它们推送到DockerHub. # 标记检查 {#marker-check} -该检查意味着集成系统已经开始处理PR.当它处于待定状态时, 意味着还没有开始所有的检查. 在所有检查启动后，它将状态更改为'成功'. +该检查意味着CI系统已经开始处理PR.当它处于'待处理'状态时，意味着尚未开始所有检查. 启动所有检查后，状态更改为'成功'. # 格式检查 {#style-check} 使用`utils/check-style/check-style`二进制文件执行一些简单的基于正则表达式的代码样式检查(注意, 它可以在本地运行). -如果失败, 按照[代码样式指南](https://clickhouse.com/docs/en/development/style/)修复样式错误. +如果失败, 按照[代码样式指南](./style.md)修复样式错误. -# 查看报告详情 {#report-details} -- [状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) -- `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) +# 报告详情 {#report-details} +- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) +- `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt) -# PVS Check {#pvs-check} -使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告找出确切的错误.如果可以则修复它们, 如果不行, 可以请ClickHouse的维护人员帮忙. +# PVS 检查 {#pvs-check} +使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告以查看确切的错误.如果可以则修复它们, 如果不行, 可以向ClickHouse的维护人员寻求帮忙. -# 查看报告详情 {#report-details} -- [状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) -- `test_run.txt.out.log`记录了构建和分析日志文件.它只包含解析或未找到的错误. -- `HTML report`记录了分析结果.关于它的描述, 请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) +# 报告详情 {#report-details} +- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) +- `test_run.txt.out.log`包含构建和分析日志文件.它只包含解析或未找到的错误. +- `HTML report`包含分析结果.有关说明请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) -# {#fast-test} -通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 通过报告查看哪些测试失败, 然后按照[这里](https://clickhouse.com/docs/en/development/tests/#functional-test-locally)描述的在本地重现失败. +# 快速测试 {#fast-test} +通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 查看报告以了解哪些测试失败, 然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败. -# 查看报告详情 {#report-details} -[状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +# 报告详情 {#report-details} +[状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/fast_test.html) # 状态页文件 {#status-page-files} - `runlog.out.log` 是包含所有其他日志的通用日志. @@ -72,27 +71,27 @@ git push - `install_log.txt` - `clickhouse-server.err.log` - `build_log.txt` -- `cmake_log.txt` 包含关于C/ c++和Linux标志检查的消息. +- `cmake_log.txt` 包含关于C/C++和Linux标志检查的消息. # 状态页列信息 {#status-page-columns} -- 测试名称 -- 包含测试的名称(没有路径, 例如, 所有类型的测试将被剥离到该名称). -- 测试状态 -- 跳过、成功或失败. -- 测试时间, 等等. -- 这个测试是空的. +- 测试名称 -- 包含测试的名称(不带路径, 例如, 所有类型的测试将被剥离到该名称). +- 测试状态 -- 跳过、成功或失败之一. +- 测试时间, 秒. -- 这个测试是空的. # 建构检查 {#build-check} -在各种配置中构建ClickHouse, 以便在后续步骤中使用. 您必须修复失败的构建.构建日志通常有足够的信息来修复错误, 但是您可能必须在本地重新生成错误. cmake选项可以在构建日志中找到, 为cmake做准备.使用这些选项并遵循一般的构建过程. +在各种配置中构建ClickHouse, 以便在后续步骤中使用. 您必须修复失败的构建.构建日志通常有足够的信息来修复错误, 但是您可能必须在本地重现故障. `cmake`选项可以在构建日志中通过grep `cmake`操作找到.使用这些选项并遵循[一般的构建过程](./build.md). -# 查看报告详情 {#report-details} -[状态页案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) -- **Compiler**: `gcc-9` or `clang-10` (or `clang-10-xx` for other architectures e.g. `clang-10-freebsd`). +# 报告详情 {#report-details} +[状态页示例](https://clickhouse-builds.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/clickhouse_build_check/report.html) +- **Compiler**: `gcc-9` 或 `clang-10` (或其他架构的`clang-10-xx`, 比如`clang-10-freebsd`). - **Build type**: `Debug` or `RelWithDebInfo` (cmake). - **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan). - **Bundled**: `bundled` 构建使用来自 `contrib` 库, 而 `unbundled` 构建使用系统库. - **Splitted**: `splitted` is a [split build](https://clickhouse.com/docs/en/development/build/#split-build) - **Status**: `成功` 或 `失败` -- **Build log**: 链接到生成和文件复制日志，这在生成失败时很有用. +- **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用. - **Build time**. -- **Artifacts**: 构建结果文件 (with `XXX` being the server version e.g. `20.8.1.4344`). +- **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`). - `clickhouse-client_XXX_all.deb` -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - `clickhouse-common-staticXXX_amd64.deb` @@ -102,15 +101,55 @@ git push - `clickhouse_XXX_amd64.changes` - `clickhouse`: Main built binary. - `clickhouse-odbc-bridge` - - `unit_tests_dbms`: GoogleTest二进制和ClickHouse单元测试. + - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件. - `shared_build.tgz`: 使用共享库构建. - `performance.tgz`: 用于性能测试的特殊包. -# 特殊的构建检查 {#special-buildcheck} -使用clang-tidy执行静态分析和代码风格检查. 报告类似于构建检查. 修复在构建日志中发现的错误. +# 特殊构建检查 {#special-buildcheck} +使用clang-tidy执行静态分析和代码样式检查. 该报告类似于构建检查. 修复在构建日志中发现的错误. +# 功能无状态测试 {#functional-stateless-tests} +为构建在不同配置中的ClickHouse二进制文件运行[无状态功能测试](./tests.md#functional-tests)——发布、调试、使用杀毒软件等.通过报告查看哪些测试失败，然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败.注意, 您必须使用正确的构建配置来重现——在AddressSanitizer下测试可能失败,但在Debug中可以通过.从[CI构建检查页面](./build.md#you-dont-have-to-build-clickhouse)下载二进制文件, 或者在本地构建它. +# 功能有状态测试 {#functional-stateful-tests} +运行[有状态功能测试](./tests.md#functional-tests).以无状态功能测试相同的方式对待它们.不同之处在于它们需要从[Yandex.Metrica数据集](https://clickhouse.com/docs/en/getting-started/example-datasets/metrica/)的`hits`和`visits`表来运行. +# 集成测试 {#integration-tests} +运行[集成测试](./tests.md#integration-tests). +# Testflows 检查{#testflows-check} +使用Testflows测试系统去运行一些测试, 在[此处](../../../tests/testflows/README.md#running-tests-locally)查看如何在本地运行它们. +# 压力测试 {#stress-test} +从多个客户端并发运行无状态功能测试, 用以检测与并发相关的错误.如果失败: +``` +* Fix all other test failures first; +* Look at the report to find the server logs and check them for possible causes + of error. +``` +# 冒烟测试 {#split-build-smoke-test} +检查[拆分构建](./build.md#split-build)配置中的服务器构建是否可以启动并运行简单查询.如果失败: +``` +* Fix other test errors first; +* Build the server in [split build](./build.md#split-build) configuration + locally and check whether it can start and run `select 1`. +``` + +# 兼容性检查 {#compatibility-check} +检查`clickhouse`二进制文件是否可以在带有旧libc版本的发行版上运行.如果失败, 请向维护人员寻求帮助. + +# AST模糊器 {#ast-fuzzer} +运行随机生成的查询来捕获程序错误.如果失败, 请向维护人员寻求帮助. + +# 性能测试 {#performance-tests} +测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](../../../docker/test/performance-comparison/README.md#how-to-read-the-report)有详细描述. + +# 质量保证 {#qa} +什么是状态页面上的任务(专用网络)项目? + +它是 Yandex 内部工作系统的链接. Yandex 员工可以看到检查的开始时间及其更详细的状态. + +运行测试的地方 + +Yandex 内部基础设施的某个地方. From 12b378f27f1da3e401bb6deab183d06a35bb446f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Sat, 23 Oct 2021 15:56:25 +0800 Subject: [PATCH 057/173] Translation for Kerberos Document: Chinese translation for Operations - External User Authenticators and Directories - Kerberos --- .../external-authenticators/kerberos.md | 106 +++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/zh/operations/external-authenticators/kerberos.md b/docs/zh/operations/external-authenticators/kerberos.md index b5a4d557de4..fb7a6c0d67a 120000 --- a/docs/zh/operations/external-authenticators/kerberos.md +++ b/docs/zh/operations/external-authenticators/kerberos.md @@ -1 +1,105 @@ -../../../en/operations/external-authenticators/kerberos.md \ No newline at end of file +# Kerberos认证 {#kerberos} +现有正确配置的 ClickHouse 用户可以通过 Kerberos 身份验证协议进行身份验证. + +目前, Kerberos 只能用作现有用户的外部身份验证器，这些用户在 `users.xml` 或本地访问控制路径中定义. +这些用户只能使用 HTTP 请求, 并且必须能够使用 GSS-SPNEGO 机制进行身份验证. + +对于这种方法, 必须在系统中配置 Kerberos, 且必须在 ClickHouse 配置中启用. + +# 开启Kerberos {#enabling-kerberos-in-clickHouse} +要启用 Kerberos, 应该在 `config.xml` 中包含 `kerberos` 部分. 此部分可能包含其他参数. + +## 参数: {#parameters} +- `principal` - 将在接受安全上下文时获取和使用的规范服务主体名称. +- 此参数是可选的, 如果省略, 将使用默认主体. + +- `realm` - 一个领域, 用于将身份验证限制为仅那些发起者领域与其匹配的请求. + +- 此参数是可选的，如果省略，则不会应用其他领域的过滤. + +示例 (进入 `config.xml`): +``` + + + + +``` + +主体规范: +``` + + + + HTTP/clickhouse.example.com@EXAMPLE.COM + + +``` + +按领域过滤: +``` + + + + EXAMPLE.COM + + +``` + +**注意** + +您只能定义一个 `kerberos` 部分. 多个 `kerberos` 部分的存在将强制 ClickHouse 禁用 Kerberos 身份验证. + +**注意** + +`主体`和`领域`部分不能同时指定. `主体`和`领域`的出现将迫使ClickHouse禁用Kerberos身份验证. + +# Kerberos作为现有用户的外部身份验证器 {#kerberos-as-an-external-authenticator-for-existing-users} +Kerberos可以用作验证本地定义用户(在`users.xml`或本地访问控制路径中定义的用户)身份的方法。目前，**只有**通过HTTP接口的请求才能被认证(通过GSS-SPNEGO机制). + +Kerberos主体名称格式通常遵循以下模式: +- *primary/instance@REALM* + +*/instance* 部分可能出现零次或多次. **发起者的规范主体名称的主要部分应与被认证用户名匹配, 以便身份验证成功**. + +# `users.xml`中启用Kerberos {#enabling-kerberos-in-users.xml} +为了启用用户的 Kerberos 身份验证, 请在用户定义中指定 `kerberos` 部分而不是`密码`或类似部分. + +参数: +- `realm` - 用于将身份验证限制为仅那些发起者的领域与其匹配的请求的领域. +- 此参数是可选的, 如果省略, 则不会应用其他按领域的过滤. + +示例 (进入 `users.xml`): +``` + + + + + + + + EXAMPLE.COM + + + + +``` + +**警告** + +注意, Kerberos身份验证不能与任何其他身份验证机制一起使用. 任何其他部分(如`密码`和`kerberos`)的出现都会迫使ClickHouse关闭. + +**提醒** + +请注意, 现在, 一旦用户 `my_user` 使用 `kerberos`, 必须在主 `config.xml` 文件中启用 Kerberos，如前所述. + +# 使用 SQL 启用 Kerberos {#enabling-kerberos-using-sql} +在 ClickHouse 中启用 [SQL 驱动的访问控制和帐户管理](./access-rights.md#access-control)后, 也可以使用 SQL 语句创建由 Kerberos 识别的用户. + +``` +CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' +``` + +...或者, 不按领域过滤: +``` +CREATE USER my_user IDENTIFIED WITH kerberos +``` From 3e6870c54004b8d1a76263078a19b8700a7bf806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Sat, 23 Oct 2021 16:24:43 +0800 Subject: [PATCH 058/173] chinese translation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adjust 事件时间 for DateTime64 to 事件时间(微秒) --- docs/zh/operations/system-tables/asynchronous_metric_log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/operations/system-tables/asynchronous_metric_log.md b/docs/zh/operations/system-tables/asynchronous_metric_log.md index d49676794fc..592fb99c5ef 100644 --- a/docs/zh/operations/system-tables/asynchronous_metric_log.md +++ b/docs/zh/operations/system-tables/asynchronous_metric_log.md @@ -10,7 +10,7 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 列： - `event_date` ([Date](../../sql-reference/data-types/date.md)) — 事件日期. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 事件时间. -- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件时间. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 事件时间(微秒). - `name` ([String](../../sql-reference/data-types/string.md)) — 指标名. - `value` ([Float64](../../sql-reference/data-types/float.md)) — 指标值. @@ -35,4 +35,4 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10 **另请参阅** - [system.asynchronous_metrics](../../operations/system-tables/asynchronous_metrics.md#system_tables-asynchronous_metrics) — 包含在后台定期计算的指标. -- [system.metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log) — 包含定期刷新到磁盘表 `system.metrics` 以及 `system.events` 中的指标值历史记录. \ No newline at end of file +- [system.metric_log](../../operations/system-tables/metric_log.md#system_tables-metric_log) — 包含定期刷新到磁盘表 `system.metrics` 以及 `system.events` 中的指标值历史记录. From 4fa3b94eada854dbd8818d17bc45cd99e1c2a942 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Sun, 24 Oct 2021 20:29:41 +0300 Subject: [PATCH 059/173] Update intersect.md --- docs/ru/sql-reference/statements/select/intersect.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/select/intersect.md b/docs/ru/sql-reference/statements/select/intersect.md index cf0c5aee55a..2206b141472 100644 --- a/docs/ru/sql-reference/statements/select/intersect.md +++ b/docs/ru/sql-reference/statements/select/intersect.md @@ -4,7 +4,7 @@ toc_title: INTERSECT # Секция INTERSECT {#intersect-clause} -`INTERSECT` возвращает только те строки, которые есть только в результатах первого и второго запросов. В запросах должны совпадать количество столбцов, их порядок и тип. Результат `INTERSECT` может содержать повторяющиеся строки. +`INTERSECT` возвращает строки, которые есть только в результатах первого и второго запросов. В запросах должны совпадать количество столбцов, их порядок и тип. Результат `INTERSECT` может содержать повторяющиеся строки. Если используется несколько `INTERSECT` и скобки не указаны, пересечение выполняется слева направо. У `INTERSECT` более высокий приоритет выполнения, чем у `UNION` и `EXCEPT`. From 37de54937cede70f5b7ce4fd6eb8a65c6c6691af Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Sun, 24 Oct 2021 22:32:28 +0300 Subject: [PATCH 060/173] fix use-after-free --- src/Interpreters/AsynchronousInsertQueue.cpp | 10 ++++++++-- ...ert_7.reference => 02015_async_inserts_7.reference} | 0 ...2015_async_insert_7.sh => 02015_async_inserts_7.sh} | 0 3 files changed, 8 insertions(+), 2 deletions(-) rename tests/queries/0_stateless/{02015_async_insert_7.reference => 02015_async_inserts_7.reference} (100%) rename tests/queries/0_stateless/{02015_async_insert_7.sh => 02015_async_inserts_7.sh} (100%) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index cc04c5f013f..2f86c8bef28 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -407,14 +407,20 @@ try } StreamingFormatExecutor executor(header, format, std::move(on_error), std::move(adding_defaults_transform)); - std::unique_ptr buffer; + std::unique_ptr last_buffer; for (const auto & entry : data->entries) { - buffer = std::make_unique(entry->bytes); + auto buffer = std::make_unique(entry->bytes); current_entry = entry; total_rows += executor.execute(*buffer); + + /// Keep buffer, because it still can be used + /// in destructor, while resetting buffer at next iteration. + last_buffer = std::move(buffer); } + format->addBuffer(std::move(last_buffer)); + auto chunk = Chunk(executor.getResultColumns(), total_rows); size_t total_bytes = chunk.bytes(); diff --git a/tests/queries/0_stateless/02015_async_insert_7.reference b/tests/queries/0_stateless/02015_async_inserts_7.reference similarity index 100% rename from tests/queries/0_stateless/02015_async_insert_7.reference rename to tests/queries/0_stateless/02015_async_inserts_7.reference diff --git a/tests/queries/0_stateless/02015_async_insert_7.sh b/tests/queries/0_stateless/02015_async_inserts_7.sh similarity index 100% rename from tests/queries/0_stateless/02015_async_insert_7.sh rename to tests/queries/0_stateless/02015_async_inserts_7.sh From fb4eb4cf2362163e12e61c05083a289b63d39491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Mon, 25 Oct 2021 11:45:24 +0800 Subject: [PATCH 061/173] chinese translation adjust title level --- .../external-authenticators/kerberos.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/zh/operations/external-authenticators/kerberos.md b/docs/zh/operations/external-authenticators/kerberos.md index fb7a6c0d67a..986795aea8b 120000 --- a/docs/zh/operations/external-authenticators/kerberos.md +++ b/docs/zh/operations/external-authenticators/kerberos.md @@ -1,4 +1,4 @@ -# Kerberos认证 {#kerberos} +# Kerberos认证 {#external-authenticators-kerberos} 现有正确配置的 ClickHouse 用户可以通过 Kerberos 身份验证协议进行身份验证. 目前, Kerberos 只能用作现有用户的外部身份验证器，这些用户在 `users.xml` 或本地访问控制路径中定义. @@ -6,10 +6,10 @@ 对于这种方法, 必须在系统中配置 Kerberos, 且必须在 ClickHouse 配置中启用. -# 开启Kerberos {#enabling-kerberos-in-clickHouse} +## 开启Kerberos {#enabling-kerberos-in-clickHouse} 要启用 Kerberos, 应该在 `config.xml` 中包含 `kerberos` 部分. 此部分可能包含其他参数. -## 参数: {#parameters} +#### 参数: {#parameters} - `principal` - 将在接受安全上下文时获取和使用的规范服务主体名称. - 此参数是可选的, 如果省略, 将使用默认主体. @@ -18,7 +18,7 @@ - 此参数是可选的，如果省略，则不会应用其他领域的过滤. 示例 (进入 `config.xml`): -``` +```xml @@ -26,7 +26,7 @@ ``` 主体规范: -``` +```xml @@ -36,7 +36,7 @@ ``` 按领域过滤: -``` +```xml @@ -53,7 +53,7 @@ `主体`和`领域`部分不能同时指定. `主体`和`领域`的出现将迫使ClickHouse禁用Kerberos身份验证. -# Kerberos作为现有用户的外部身份验证器 {#kerberos-as-an-external-authenticator-for-existing-users} +## Kerberos作为现有用户的外部身份验证器 {#kerberos-as-an-external-authenticator-for-existing-users} Kerberos可以用作验证本地定义用户(在`users.xml`或本地访问控制路径中定义的用户)身份的方法。目前，**只有**通过HTTP接口的请求才能被认证(通过GSS-SPNEGO机制). Kerberos主体名称格式通常遵循以下模式: @@ -61,7 +61,7 @@ Kerberos主体名称格式通常遵循以下模式: */instance* 部分可能出现零次或多次. **发起者的规范主体名称的主要部分应与被认证用户名匹配, 以便身份验证成功**. -# `users.xml`中启用Kerberos {#enabling-kerberos-in-users.xml} +### `users.xml`中启用Kerberos {#enabling-kerberos-in-users-xml} 为了启用用户的 Kerberos 身份验证, 请在用户定义中指定 `kerberos` 部分而不是`密码`或类似部分. 参数: @@ -92,14 +92,14 @@ Kerberos主体名称格式通常遵循以下模式: 请注意, 现在, 一旦用户 `my_user` 使用 `kerberos`, 必须在主 `config.xml` 文件中启用 Kerberos，如前所述. -# 使用 SQL 启用 Kerberos {#enabling-kerberos-using-sql} +### 使用 SQL 启用 Kerberos {#enabling-kerberos-using-sql} 在 ClickHouse 中启用 [SQL 驱动的访问控制和帐户管理](./access-rights.md#access-control)后, 也可以使用 SQL 语句创建由 Kerberos 识别的用户. -``` +```sql CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' ``` ...或者, 不按领域过滤: -``` +```sql CREATE USER my_user IDENTIFIED WITH kerberos ``` From 171c5845ed7c84dbacdb242eab4c38fbfddc156f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Mon, 25 Oct 2021 14:13:03 +0800 Subject: [PATCH 062/173] chinese translation adjust waning module --- docs/zh/operations/external-authenticators/kerberos.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh/operations/external-authenticators/kerberos.md b/docs/zh/operations/external-authenticators/kerberos.md index 986795aea8b..1121a86a793 120000 --- a/docs/zh/operations/external-authenticators/kerberos.md +++ b/docs/zh/operations/external-authenticators/kerberos.md @@ -45,11 +45,11 @@ ``` -**注意** +!!! warning "注意" 您只能定义一个 `kerberos` 部分. 多个 `kerberos` 部分的存在将强制 ClickHouse 禁用 Kerberos 身份验证. -**注意** +!!! warning "注意" `主体`和`领域`部分不能同时指定. `主体`和`领域`的出现将迫使ClickHouse禁用Kerberos身份验证. @@ -84,11 +84,11 @@ Kerberos主体名称格式通常遵循以下模式: ``` -**警告** +!!! warning "警告" 注意, Kerberos身份验证不能与任何其他身份验证机制一起使用. 任何其他部分(如`密码`和`kerberos`)的出现都会迫使ClickHouse关闭. -**提醒** +!!! info "提醒" 请注意, 现在, 一旦用户 `my_user` 使用 `kerberos`, 必须在主 `config.xml` 文件中启用 Kerberos，如前所述. From db47dcf14fdd4122ba06b3ab6479a13ebe7b8930 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 25 Oct 2021 11:18:00 +0300 Subject: [PATCH 063/173] Added test. --- ...er_table_alter_block_structure_2.reference | 3 +++ ...6_buffer_table_alter_block_structure_2.sql | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.reference create mode 100644 tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql diff --git a/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.reference b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.reference new file mode 100644 index 00000000000..1f90610041b --- /dev/null +++ b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.reference @@ -0,0 +1,3 @@ +2020-01-01 00:05:00 +2020-01-01 00:05:00 +2020-01-01 00:06:00 hello diff --git a/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql new file mode 100644 index 00000000000..8862037c82b --- /dev/null +++ b/tests/queries/0_stateless/01506_buffer_table_alter_block_structure_2.sql @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS buf_dest; +DROP TABLE IF EXISTS buf; + +CREATE TABLE buf_dest (timestamp DateTime) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (timestamp); + +CREATE TABLE buf (timestamp DateTime) Engine = Buffer(currentDatabase(), buf_dest, 16, 0.1, 0.1, 2000000, 20000000, 100000000, 300000000);; + +INSERT INTO buf (timestamp) VALUES (toDateTime('2020-01-01 00:05:00')); + +--- wait for buffer to flush +SELECT sleep(1) from numbers(1) settings max_block_size=1 format Null; + +ALTER TABLE buf_dest ADD COLUMN s String; +ALTER TABLE buf ADD COLUMN s String; + +SELECT * FROM buf; + +INSERT INTO buf (timestamp, s) VALUES (toDateTime('2020-01-01 00:06:00'), 'hello'); + +SELECT * FROM buf ORDER BY timestamp; + +DROP TABLE IF EXISTS buf; +DROP TABLE IF EXISTS buf_dest; From a0a5c1a21db6114936f62190249a61da1f68b233 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 25 Oct 2021 12:06:53 +0300 Subject: [PATCH 064/173] Fix argument types for now and now64 --- src/Functions/now.cpp | 11 +++++++---- src/Functions/now64.cpp | 14 ++++++++++---- .../0_stateless/02100_now64_types_bug.reference | 3 +++ .../queries/0_stateless/02100_now64_types_bug.sql | 8 ++++++++ 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02100_now64_types_bug.reference create mode 100644 tests/queries/0_stateless/02100_now64_types_bug.sql diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp index b244d054cfe..653a39b38d0 100644 --- a/src/Functions/now.cpp +++ b/src/Functions/now.cpp @@ -43,13 +43,13 @@ private: class FunctionBaseNow : public IFunctionBase { public: - explicit FunctionBaseNow(time_t time_, DataTypePtr return_type_) : time_value(time_), return_type(return_type_) {} + explicit FunctionBaseNow(time_t time_, DataTypes argument_types_, DataTypePtr return_type_) + : time_value(time_), argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {} String getName() const override { return "now"; } const DataTypes & getArgumentTypes() const override { - static const DataTypes argument_types; return argument_types; } @@ -69,6 +69,7 @@ public: private: time_t time_value; + DataTypes argument_types; DataTypePtr return_type; }; @@ -117,8 +118,10 @@ public: } if (arguments.size() == 1) return std::make_unique( - time(nullptr), std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 0, 0))); - return std::make_unique(time(nullptr), std::make_shared()); + time(nullptr), DataTypes{arguments.front().type}, + std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 0, 0))); + + return std::make_unique(time(nullptr), DataTypes(), std::make_shared()); } }; diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index 4a3f4dbfb22..bd1038b1fc6 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -67,13 +67,13 @@ private: class FunctionBaseNow64 : public IFunctionBase { public: - explicit FunctionBaseNow64(Field time_, DataTypePtr return_type_) : time_value(time_), return_type(return_type_) {} + explicit FunctionBaseNow64(Field time_, DataTypes argument_types_, DataTypePtr return_type_) + : time_value(time_), argument_types(std::move(argument_types_)), return_type(std::move(return_type_)) {} String getName() const override { return "now64"; } const DataTypes & getArgumentTypes() const override { - static const DataTypes argument_types; return argument_types; } @@ -93,6 +93,7 @@ public: private: Field time_value; + DataTypes argument_types; DataTypePtr return_type; }; @@ -139,14 +140,19 @@ public: return std::make_shared(scale, timezone_name); } - FunctionBasePtr buildImpl(const ColumnsWithTypeAndName &, const DataTypePtr & result_type) const override + FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override { UInt32 scale = DataTypeDateTime64::default_scale; auto res_type = removeNullable(result_type); if (const auto * type = typeid_cast(res_type.get())) scale = type->getScale(); - return std::make_unique(nowSubsecond(scale), result_type); + DataTypes arg_types; + arg_types.reserve(arguments.size()); + for (const auto & arg : arguments) + arg_types.push_back(arg.type); + + return std::make_unique(nowSubsecond(scale), std::move(arg_types), std::move(result_type)); } }; diff --git a/tests/queries/0_stateless/02100_now64_types_bug.reference b/tests/queries/0_stateless/02100_now64_types_bug.reference new file mode 100644 index 00000000000..ae136939b64 --- /dev/null +++ b/tests/queries/0_stateless/02100_now64_types_bug.reference @@ -0,0 +1,3 @@ +2 +1 +0 diff --git a/tests/queries/0_stateless/02100_now64_types_bug.sql b/tests/queries/0_stateless/02100_now64_types_bug.sql new file mode 100644 index 00000000000..c91defcbf3a --- /dev/null +++ b/tests/queries/0_stateless/02100_now64_types_bug.sql @@ -0,0 +1,8 @@ +SELECT x +FROM +( + SELECT if((number % NULL) = -2147483648, NULL, if(toInt64(toInt64(now64(if((number % NULL) = -2147483648, NULL, if(toInt64(now64(toInt64(9223372036854775807, now64(h3kRing(NULL, NULL))), h3kRing(NULL, NULL))) = (number % NULL), nan, toFloat64(number))), toInt64(9223372036854775807, toInt64(9223372036854775807, now64(h3kRing(NULL, NULL))), now64(h3kRing(NULL, NULL))), h3kRing(NULL, NULL))), now64(toInt64(9223372036854775807, toInt64(0, now64(h3kRing(NULL, NULL))), now64(h3kRing(NULL, NULL))), h3kRing(NULL, NULL))) = (number % NULL), nan, toFloat64(number))) AS x + FROM system.numbers + LIMIT 3 +) +ORDER BY x DESC NULLS LAST From 2c20fbe5715c78effc9c8ba6989d6c3dfcf8740d Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 25 Oct 2021 12:46:50 +0300 Subject: [PATCH 065/173] Add ru translation for UTF normalized forms. --- .../functions/string-functions.md | 26 ++-- .../functions/string-functions.md | 144 ++++++++++++++++++ 2 files changed, 156 insertions(+), 14 deletions(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index f6dbadb875a..751b726765b 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -835,14 +835,14 @@ Type: [String](../../sql-reference/data-types/string.md). Query: ``` sql -SELECT length('ё'), normalizeUTF8NFC('ё') AS nfc, length(nfc) AS nfc_len; +SELECT length('â'), normalizeUTF8NFC('â') AS nfc, length(nfc) AS nfc_len; ``` Result: ``` text -┌─length('ё')─┬─nfc─┬─nfc_len─┐ -│ 2 │ ё │ 2 │ +┌─length('â')─┬─nfc─┬─nfc_len─┐ +│ 2 │ â │ 2 │ └─────────────┴─────┴─────────┘ ``` @@ -871,14 +871,14 @@ Type: [String](../../sql-reference/data-types/string.md). Query: ``` sql -SELECT length('ё'), normalizeUTF8NFD('ё') AS nfd, length(nfd) AS nfd_len; +SELECT length('â'), normalizeUTF8NFD('â') AS nfd, length(nfd) AS nfd_len; ``` Result: ``` text -┌─length('ё')─┬─nfd─┬─nfd_len─┐ -│ 2 │ ё │ 4 │ +┌─length('â')─┬─nfd─┬─nfd_len─┐ +│ 2 │ â │ 3 │ └─────────────┴─────┴─────────┘ ``` @@ -907,21 +907,19 @@ Type: [String](../../sql-reference/data-types/string.md). Query: ``` sql -SELECT length('ё'), normalizeUTF8NFKC('ё') AS nfkc, length(nfkc) AS nfkc_len; +SELECT length('â'), normalizeUTF8NFKC('â') AS nfkc, length(nfkc) AS nfkc_len; ``` Result: ``` text -┌─length('ё')─┬─nfkc─┬─nfkc_len─┐ -│ 2 │ ё │ 2 │ +┌─length('â')─┬─nfkc─┬─nfkc_len─┐ +│ 2 │ â │ 2 │ └─────────────┴──────┴──────────┘ - ``` ## normalizeUTF8NFKD {#normalizeutf8nfkd} - Converts a string to [NFKD normalized form](https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms), assuming the string contains a set of bytes that make up a UTF-8 encoded text. **Syntax** @@ -945,14 +943,14 @@ Type: [String](../../sql-reference/data-types/string.md). Query: ``` sql -SELECT length('ё'), normalizeUTF8NFKD('ё') AS nfkd, length(nfkd) AS nfkd_len; +SELECT length('â'), normalizeUTF8NFKD('â') AS nfkd, length(nfkd) AS nfkd_len; ``` Result: ``` text -┌─length('ё')─┬─nfkd─┬─nfkd_len─┐ -│ 2 │ ё │ 4 │ +┌─length('â')─┬─nfkd─┬─nfkd_len─┐ +│ 2 │ â │ 3 │ └─────────────┴──────┴──────────┘ ``` diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 95e265f0b54..f3c627541ce 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -781,6 +781,150 @@ SELECT normalizedQueryHash('SELECT 1 AS `xyz`') != normalizedQueryHash('SELECT 1 └─────┘ ``` +## normalizeUTF8NFC {#normalizeutf8nfc} + +Преобразует строку в нормализованную форму [NFC](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8. + +**Синтаксис** + +``` sql +normalizeUTF8NFKD(words) +``` + +**Аргументы** + +- `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Строка, преобразованная в нормализированную форму NFC. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +``` sql +SELECT length('â'), normalizeUTF8NFC('â') AS nfc, length(nfc) AS nfc_len; +``` + +Результат: + +``` text +┌─length('â')─┬─nfc─┬─nfc_len─┐ +│ 2 │ â │ 2 │ +└─────────────┴─────┴─────────┘ +``` + +## normalizeUTF8NFD {#normalizeutf8nfd} + +Преобразует строку в нормализованную форму [NFD](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8. + +**Синтаксис** + +``` sql +normalizeUTF8NFD(words) +``` + +**Аргументы** + +- `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Строка, преобразованная в нормализированную форму NFD. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +``` sql +SELECT length('â'), normalizeUTF8NFD('â') AS nfd, length(nfd) AS nfd_len; +``` + +Результат: + +``` text +┌─length('â')─┬─nfd─┬─nfd_len─┐ +│ 2 │ â │ 3 │ +└─────────────┴─────┴─────────┘ +``` + +## normalizeUTF8NFKC {#normalizeutf8nfkc} + +Преобразует строку в нормализованную форму [NFKC](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8. + +**Синтаксис** + +``` sql +normalizeUTF8NFKC(words) +``` + +**Аргументы** + +- `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Строка, преобразованная в нормализированную форму NFKC. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +``` sql +SELECT length('â'), normalizeUTF8NFKC('â') AS nfkc, length(nfkc) AS nfkc_len; +``` + +Результат: + +``` text +┌─length('â')─┬─nfkc─┬─nfkc_len─┐ +│ 2 │ â │ 2 │ +└─────────────┴──────┴──────────┘ +``` + +## normalizeUTF8NFKD {#normalizeutf8nfkd} + +Преобразует строку в нормализованную форму [NFKD](https://ru.wikipedia.org/wiki/Юникод#Алгоритмы_нормализации), предполагая, что строка содержит набор байтов, составляющих текст в кодировке UTF-8. + +**Синтаксис** + +``` sql +normalizeUTF8NFKD(words) +``` + +**Аргументы** + +- `words` — входная строка, которая содержит набор байтов, составляющих текст в кодировке UTF-8. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Строка, преобразованная в нормализированную форму NFKD. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Запрос: + +``` sql +SELECT length('â'), normalizeUTF8NFKD('â') AS nfkd, length(nfkd) AS nfkd_len; +``` + +Результат: + +``` text +┌─length('â')─┬─nfkd─┬─nfkd_len─┐ +│ 2 │ â │ 3 │ +└─────────────┴──────┴──────────┘ +``` + ## encodeXMLComponent {#encode-xml-component} Экранирует символы для размещения строки в текстовом узле или атрибуте XML. From 264e173685c4abddfc1aa321eda5bbd1d36f9b9e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 25 Oct 2021 13:04:16 +0300 Subject: [PATCH 066/173] Update 02100_now64_types_bug.sql --- tests/queries/0_stateless/02100_now64_types_bug.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02100_now64_types_bug.sql b/tests/queries/0_stateless/02100_now64_types_bug.sql index c91defcbf3a..ef91b19bb08 100644 --- a/tests/queries/0_stateless/02100_now64_types_bug.sql +++ b/tests/queries/0_stateless/02100_now64_types_bug.sql @@ -1,7 +1,7 @@ SELECT x FROM ( - SELECT if((number % NULL) = -2147483648, NULL, if(toInt64(toInt64(now64(if((number % NULL) = -2147483648, NULL, if(toInt64(now64(toInt64(9223372036854775807, now64(h3kRing(NULL, NULL))), h3kRing(NULL, NULL))) = (number % NULL), nan, toFloat64(number))), toInt64(9223372036854775807, toInt64(9223372036854775807, now64(h3kRing(NULL, NULL))), now64(h3kRing(NULL, NULL))), h3kRing(NULL, NULL))), now64(toInt64(9223372036854775807, toInt64(0, now64(h3kRing(NULL, NULL))), now64(h3kRing(NULL, NULL))), h3kRing(NULL, NULL))) = (number % NULL), nan, toFloat64(number))) AS x + SELECT if((number % NULL) = -2147483648, NULL, if(toInt64(toInt64(now64(if((number % NULL) = -2147483648, NULL, if(toInt64(now64(toInt64(9223372036854775807, now64(plus(NULL, NULL))), plus(NULL, NULL))) = (number % NULL), nan, toFloat64(number))), toInt64(9223372036854775807, toInt64(9223372036854775807, now64(plus(NULL, NULL))), now64(plus(NULL, NULL))), plus(NULL, NULL))), now64(toInt64(9223372036854775807, toInt64(0, now64(plus(NULL, NULL))), now64(plus(NULL, NULL))), plus(NULL, NULL))) = (number % NULL), nan, toFloat64(number))) AS x FROM system.numbers LIMIT 3 ) From 538d71b855aa1c2e36c586eab838d409ba2ffbe4 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 25 Oct 2021 13:44:46 +0300 Subject: [PATCH 067/173] Update docs/ru/sql-reference/functions/string-functions.md Co-authored-by: gyuton <40863448+gyuton@users.noreply.github.com> --- docs/ru/sql-reference/functions/string-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index f3c627541ce..10ecc6c0a83 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -788,7 +788,7 @@ SELECT normalizedQueryHash('SELECT 1 AS `xyz`') != normalizedQueryHash('SELECT 1 **Синтаксис** ``` sql -normalizeUTF8NFKD(words) +normalizeUTF8NFC(words) ``` **Аргументы** From a211e48c3c1e910d2ecac1327f51cd368ddf0a9f Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 25 Oct 2021 13:44:50 +0300 Subject: [PATCH 068/173] Update docs/en/sql-reference/functions/string-functions.md Co-authored-by: gyuton <40863448+gyuton@users.noreply.github.com> --- docs/en/sql-reference/functions/string-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 751b726765b..4391d013440 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -817,7 +817,7 @@ Converts a string to [NFC normalized form](https://en.wikipedia.org/wiki/Unicode **Syntax** ``` sql -normalizeUTF8NFKD(words) +normalizeUTF8NFC(words) ``` **Arguments** From 8ec5d68ce3d3ef1ca023db32618daf2acecf1239 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Mon, 25 Oct 2021 15:02:24 +0300 Subject: [PATCH 069/173] Apply suggestions from code review Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/operations/settings/settings.md | 6 +++--- docs/en/sql-reference/statements/insert-into.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f02fe383f37..3eb39907991 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3817,7 +3817,7 @@ Default value: `0`. ## async_insert {#async-insert} -Enables or disables asynchronous inserts for multiple concurrent connetions. This makes sense only for insertion over HTTP protocol. Note, that deduplications isn't working for such inserts. +Enables or disables asynchronous inserts for multiple concurrent connections. This makes sense only for insertion over HTTP protocol. Note that deduplication isn't working for such inserts. If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables. @@ -3827,8 +3827,8 @@ If [wait_for_async_insert](#wait-for-async-insert) is enabled, every client will Possible values: -- 0 — Disabled. -- 1 — Enabled. +- 0 — Insertions are made synchronously, one after another. +- 1 — Multiple asynchronous insertions enabled. Default value: `0`. diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 83812a198d9..c44f7435089 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -119,4 +119,4 @@ Performance will not decrease if: - Data is added in real time. - You upload data that is usually sorted by time. -It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections with the setting [async_insert](../../operations/settings/settings.md#async-insert). The data from such inserts is combined into batches and then safely inserted into a table. +It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections. The data from such insertions is combined into batches and then safely inserted into a table. To enable the asynchronous mode, switch on the [async_insert](../../operations/settings/settings.md#async-insert) setting. Note that asynchronous insertions are supported only over HTTP protocol, and deduplication is not supported for them. From 0642aeca3a9e71b74912ca3e58533f1c724aa57d Mon Sep 17 00:00:00 2001 From: gyuton Date: Mon, 25 Oct 2021 15:55:04 +0300 Subject: [PATCH 070/173] Revert "Merge branch 'gyuton-DOCSUP-16593-document-tuple-functions-and-operators' of https://github.com/gyuton/ClickHouse into gyuton-DOCSUP-14941-Document-asynchronous-insert-mode" This reverts commit ccad91b5effbc3d1d9d5007b01e6a011ac950dab, reversing changes made to 4ecd2ed6e204d4d70200810c43d5370dcbd61ee9. --- .../functions/arithmetic-functions.md | 73 -- .../functions/tuple-functions.md | 801 ----------------- docs/en/sql-reference/operators/index.md | 10 - .../functions/arithmetic-functions.md | 75 -- .../functions/tuple-functions.md | 802 ------------------ docs/ru/sql-reference/operators/index.md | 10 - 6 files changed, 1771 deletions(-) diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md index 40fadf34eab..3187f13b5b9 100644 --- a/docs/en/sql-reference/functions/arithmetic-functions.md +++ b/docs/en/sql-reference/functions/arithmetic-functions.md @@ -82,76 +82,3 @@ An exception is thrown when dividing by zero or when dividing a minimal negative Returns the least common multiple of the numbers. An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. -## max2 {#max2} - -Compares two values and returns the maximum. The returned value is converted to [Float64](../../sql-reference/data-types/float.md). - -**Syntax** - -```sql -max2(value1, value2) -``` - -**Arguments** - -- `value1` — First value. [Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). -- `value2` — Second value. [Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- The maximum of two values. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT max2(-1, 2); -``` - -Result: - -```text -┌─max2(-1, 2)─┐ -│ 2 │ -└─────────────┘ -``` - -## min2 {#min2} - -Compares two values and returns the minimum. The returned value is converted to [Float64](../../sql-reference/data-types/float.md). - -**Syntax** - -```sql -max2(value1, value2) -``` - -**Arguments** - -- `value1` — First value. [Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). -- `value2` — Second value. [Int/UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- The minimum of two values. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT min2(-1, 2); -``` - -Result: - -```text -┌─min2(-1, 2)─┐ -│ -1 │ -└─────────────┘ -``` diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 73ebd9ad7d6..39e59ae2ba9 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -165,804 +165,3 @@ Result: │ 2 │ └─────────────────┘ ``` - -## tuplePlus {#tupleplus} - -Calculates the sum of corresponding values of two tuples of the same size. - -**Syntax** - -```sql -tuplePlus(tuple1, tuple2) -``` - -Alias: `vectorSum`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Tuple with the sum. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tuplePlus((1, 2), (2, 3)); -``` - -Result: - -```text -┌─tuplePlus((1, 2), (2, 3))─┐ -│ (3,5) │ -└───────────────────────────┘ -``` - -## tupleMinus {#tupleminus} - -Calculates the subtraction of corresponding values of two tuples of the same size. - -**Syntax** - -```sql -tupleMinus(tuple1, tuple2) -``` - -Alias: `vectorDifference`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Tuple with the result of subtraction. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tupleMinus((1, 2), (2, 3)); -``` - -Result: - -```text -┌─tupleMinus((1, 2), (2, 3))─┐ -│ (-1,-1) │ -└────────────────────────────┘ -``` - -## tupleMultiply {#tuplemultiply} - -Calculates the multiplication of corresponding values of two tuples of the same size. - -**Syntax** - -```sql -tupleMultiply(tuple1, tuple2) -``` - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Tuple with the multiplication. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tupleMultiply((1, 2), (2, 3)); -``` - -Result: - -```text -┌─tupleMultiply((1, 2), (2, 3))─┐ -│ (2,6) │ -└───────────────────────────────┘ -``` - -## tupleDivide {#tupledivide} - -Calculates the division of corresponding values of two tuples of the same size. Note that division by zero will return `inf`. - -**Syntax** - -```sql -tupleDivide(tuple1, tuple2) -``` - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Tuple with the result of division. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tupleDivide((1, 2), (2, 3)); -``` - -Result: - -```text -┌─tupleDivide((1, 2), (2, 3))─┐ -│ (0.5,0.6666666666666666) │ -└─────────────────────────────┘ -``` - -## tupleNegate {#tuplenegate} - -Calculates the negation of the tuple values. - -**Syntax** - -```sql -tupleNegate(tuple) -``` - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Tuple with the result of negation. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tupleDivide((1, 2)); -``` - -Result: - -```text -┌─tupleNegate((1, 2))─┐ -│ (-1,-2) │ -└─────────────────────┘ -``` - -## tupleMultiplyByNumber {#tuplemultiplybynumber} - -Returns a tuple with all values multiplied by a number. - -**Syntax** - -```sql -tupleMultiplyByNumber(tuple, number) -``` - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). -- `number` — Multiplier. [Int/UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). - -**Returned value** - -- Tuple with multiplied values. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tupleMultiplyByNumber((1, 2), -2.1); -``` - -Result: - -```text -┌─tupleMultiplyByNumber((1, 2), -2.1)─┐ -│ (-2.1,-4.2) │ -└─────────────────────────────────────┘ -``` - -## tupleDivideByNumber {#tupledividebynumber} - -Returns a tuple with all values divided by a number. Note that division by zero will return `inf`. - -**Syntax** - -```sql -tupleDivideByNumber(tuple, number) -``` - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). -- `number` — Divider. [Int/UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). - -**Returned value** - -- Tuple with divided values. - -Type: [Tuple](../../sql-reference/data-types/tuple.md). - -**Example** - -Query: - -```sql -SELECT tupleDivideByNumber((1, 2), 0.5); -``` - -Result: - -```text -┌─tupleDivideByNumber((1, 2), 0.5)─┐ -│ (2,4) │ -└──────────────────────────────────┘ -``` - -## dotProduct {#dotproduct} - -Calculates the scalar product of two tuples of the same size. - -**Syntax** - -```sql -dotProduct(tuple1, tuple2) -``` - -Alias: `scalarProduct`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Scalar product. - -Type: [Int/UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). - -**Example** - -Query: - -```sql -SELECT dotProduct((1, 2), (2, 3)); -``` - -Result: - -```text -┌─dotProduct((1, 2), (2, 3))─┐ -│ 8 │ -└────────────────────────────┘ -``` - -## L1Norm {#l1norm} - -Calculates the sum of absolute values of a tuple. - -**Syntax** - -```sql -L1Norm(tuple) -``` - -Alias: `normL1`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- L1-norm or [taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance. - -Type: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). - -**Example** - -Query: - -```sql -SELECT L1Norm((1, 2)); -``` - -Result: - -```text -┌─L1Norm((1, 2))─┐ -│ 3 │ -└────────────────┘ -``` - -## L2Norm {#l2norm} - -Calculates the square root of the sum of the squares of the tuple values. - -**Syntax** - -```sql -L2Norm(tuple) -``` - -Alias: `normL2`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- L2-norm or [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance). - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2Norm((1, 2)); -``` - -Result: - -```text -┌───L2Norm((1, 2))─┐ -│ 2.23606797749979 │ -└──────────────────┘ -``` - -## LinfNorm {#linfnorm} - -Calculates the maximum of absolute values of a tuple. - -**Syntax** - -```sql -LinfNorm(tuple) -``` - -Alias: `normLinf`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Linf-norm or the maximum absolute value. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LinfNorm((1, -2)); -``` - -Result: - -```text -┌─LinfNorm((1, -2))─┐ -│ 2 │ -└───────────────────┘ -``` - -## LpNorm {#lpnorm} - -Calculates the root of `p`-th power of the sum of the absolute values of a tuple in the power of `p`. - -**Syntax** - -```sql -LpNorm(tuple, p) -``` - -Alias: `normLp`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- [Lp-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm) - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LpNorm((1, -2),2); -``` - -Result: - -```text -┌─LpNorm((1, -2), 2)─┐ -│ 2.23606797749979 │ -└────────────────────┘ -``` - -## L1Distance {#l1distance} - -Calculates the distance between two points (the values of the tuples are the coordinates) in 1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry) distance). - -**Syntax** - -```sql -L1Distance(tuple1, tuple2) -``` - -Alias: `distanceL1`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- 1-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L1Distance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─L1Distance((1, 2), (2, 3))─┐ -│ 2 │ -└────────────────────────────┘ -``` - -## L2Distance {#l2distance} - -Calculates the distance between two points (the values of the tuples are the coordinates) in 2-norm ([Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance)). - -**Syntax** - -```sql -L2Distance(tuple1, tuple2) -``` - -Alias: `distanceL2`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- 2-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2Distance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─L2Distance((1, 2), (2, 3))─┐ -│ 1.4142135623730951 │ -└────────────────────────────┘ -``` - -## LinfDistance {#linfdistance} - -Calculates the distance between two points (the values of the tuples are the coordinates) in [infinity-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm)). - -**Syntax** - -```sql -LinfDistance(tuple1, tuple2) -``` - -Alias: `distanceLinf`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Infinity-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LinfDistance((1, 2), (2, 3)); -``` - -Result: - -```text -┌─LinfDistance((1, 2), (2, 3))─┐ -│ 1 │ -└──────────────────────────────┘ -``` - -## LpDistance {#lpdistance} - -Calculates the distance between two points (the values of the tuples are the coordinates) in [p-norm distance](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm). - -**Syntax** - -```sql -LpDistance(tuple1, tuple2, p) -``` - -Alias: `distanceLp`. - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple1` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- p-norm distance. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LpDistance((1, 2), (2, 3), 3); -``` - -Result: - -```text -┌─LpDistance((1, 2), (2, 3), 3)─┐ -│ 1.2599210498948732 │ -└───────────────────────────────┘ -``` - -## L1Normalize {#l1normalize} - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in 1-norm ([taxicab geometry](https://en.wikipedia.org/wiki/Taxicab_geometry)). - -**Syntax** - -```sql -L1Normalize(tuple) -``` - -Alias: `normalizeL1`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L1Normalize((1, 2)); -``` - -Result: - -```text -┌─L1Normalize((1, 2))─────────────────────┐ -│ (0.3333333333333333,0.6666666666666666) │ -└─────────────────────────────────────────┘ -``` - -## L2Normalize {#l2normalize} - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in 2-norm ([Euclidean geometry](https://en.wikipedia.org/wiki/Euclidean_space)). - -**Syntax** - -```sql -L2Normalize(tuple) -``` - -Alias: `normalizeL1`. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT L2Normalize((3, 4)); -``` - -Result: - -```text -┌─L2Normalize((3, 4))─┐ -│ (0.6,0.8) │ -└─────────────────────┘ -``` - -## LinfNormalize {#linfnormalize} - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in [infinity-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#Maximum_norm_(special_case_of:_infinity_norm,_uniform_norm,_or_supremum_norm)). - -**Syntax** - -```sql -LinfNormalize(tuple) -``` - -Alias: `normalizeLinf `. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LinfNormalize((3, 4)); -``` - -Result: - -```text -┌─LinfNormalize((3, 4))─┐ -│ (0.75,1) │ -└───────────────────────┘ -``` - -## LpNormalize {#lpnormalize} - -Calculates the unit vector of a given vector (the values of the tuple are the coordinates) in [p-norm](https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm). - -**Syntax** - -```sql -LpNormalize(tuple, p) -``` - -Alias: `normalizeLp `. - -**Arguments** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — The power. Possible values: any number from [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) or [Float](../../sql-reference/data-types/float.md). - -**Returned value** - -- Unit vector. - -Type: [Tuple](../../sql-reference/data-types/tuple.md) of [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT LpNormalize((3, 4),5); -``` - -Result: - -```text -┌─LpNormalize((3, 4), 5)──────────────────┐ -│ (0.7187302630182624,0.9583070173576831) │ -└─────────────────────────────────────────┘ -``` - -## cosineDistance {#cosinedistance} - -Calculates the cosine distance between two vectors (the values of the tuples are the coordinates). The less the returned value is, the more similar are the vectors. - -**Syntax** - -```sql -cosineDistance(tuple1, tuple2) -``` - -**Arguments** - -- `tuple1` — First tuple. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — Second tuple. [Tuple](../../sql-reference/data-types/tuple.md). - -**Returned value** - -- Cosine of the angle between two vectors substracted from one. - -Type: [Float](../../sql-reference/data-types/float.md). - -**Example** - -Query: - -```sql -SELECT cosineDistance((1, 2),(2,3)); -``` - -Result: - -```text -┌─cosineDistance((1, 2), (2, 3))─┐ -│ 0.007722123286332261 │ -└────────────────────────────────┘ -``` diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index aa9b7f3ba63..dbda5f5dd69 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -17,30 +17,20 @@ ClickHouse transforms operators to their corresponding functions at the query pa `-a` – The `negate (a)` function. -For tuple negation: [tupleNegate](../../sql-reference/functions/tuple-functions.md#tuplenegate). - ## Multiplication and Division Operators {#multiplication-and-division-operators} `a * b` – The `multiply (a, b)` function. -For multiplying tuple by number: [tupleMultiplyByNumber](../../sql-reference/functions/tuple-functions.md#tuplemultiplybynumber), for scalar profuct: [dotProduct](../../sql-reference/functions/tuple-functions.md#dotproduct). - `a / b` – The `divide(a, b)` function. -For dividing tuple by number: [tupleDivideByNumber](../../sql-reference/functions/tuple-functions.md#tupledividebynumber). - `a % b` – The `modulo(a, b)` function. ## Addition and Subtraction Operators {#addition-and-subtraction-operators} `a + b` – The `plus(a, b)` function. -For tuple addiction: [tuplePlus](../../sql-reference/functions/tuple-functions.md#tupleplus). - `a - b` – The `minus(a, b)` function. -For tuple subtraction: [tupleMinus](../../sql-reference/functions/tuple-functions.md#tupleminus). - ## Comparison Operators {#comparison-operators} `a = b` – The `equals(a, b)` function. diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index 278f3edb74a..f587b7b5b5d 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -83,78 +83,3 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 Вычисляет наименьшее общее кратное чисел. При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. - -## max2 {#max2} - -Сравнивает два числа и возвращает максимум. Возвращаемое значение приводится к типу [Float64](../../sql-reference/data-types/float.md). - -**Синтаксис** - -```sql -max2(value1, value2) -``` - -**Аргументы** - -- `value1` — первое число. [Int/UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). -- `value2` — второе число. [Int/UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). - -**Возвращаемое значение** - -- Максимальное значение среди двух чисел. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT max2(-1, 2); -``` - -Результат: - -```text -┌─max2(-1, 2)─┐ -│ 2 │ -└─────────────┘ -``` - -## min2 {#min2} - -Сравнивает два числа и возвращает минимум. Возвращаемое значение приводится к типу [Float64](../../sql-reference/data-types/float.md). - -**Синтаксис** - -```sql -min2(value1, value2) -``` - -**Аргументы** - -- `value1` — первое число. [Int/UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). -- `value2` — второе число. [Int/UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). - -**Возвращаемое значение** - -- Минимальное значение среди двух чисел. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT min2(-1, 2); -``` - -Результат: - -```text -┌─min2(-1, 2)─┐ -│ -1 │ -└─────────────┘ -``` - diff --git a/docs/ru/sql-reference/functions/tuple-functions.md b/docs/ru/sql-reference/functions/tuple-functions.md index 9ee3f413577..381743a450b 100644 --- a/docs/ru/sql-reference/functions/tuple-functions.md +++ b/docs/ru/sql-reference/functions/tuple-functions.md @@ -163,805 +163,3 @@ SELECT tupleHammingDistance(wordShingleMinHash(string), wordShingleMinHashCaseIn │ 2 │ └─────────────────┘ ``` - -## tuplePlus {#tupleplus} - -Вычисляет сумму соответствующих значений двух кортежей одинакового размера. - -**Синтаксис** - -```sql -tuplePlus(tuple1, tuple2) -``` - -Синоним: `vectorSum`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Кортеж с суммами. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tuplePlus((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─tuplePlus((1, 2), (2, 3))─┐ -│ (3,5) │ -└───────────────────────────┘ -``` - -## tupleMinus {#tupleminus} - -Вычисляет разность соответствующих значений двух кортежей одинакового размера. - -**Синтаксис** - -```sql -tupleMinus(tuple1, tuple2) -``` - -Синоним: `vectorDifference`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Кортеж с разностями. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tupleMinus((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─tupleMinus((1, 2), (2, 3))─┐ -│ (-1,-1) │ -└────────────────────────────┘ -``` - -## tupleMultiply {#tuplemultiply} - -Вычисляет произведение соответствующих значений двух кортежей одинакового размера. - -**Синтаксис** - -```sql -tupleMultiply(tuple1, tuple2) -``` - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Кортеж с произведениями. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tupleMultiply((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─tupleMultiply((1, 2), (2, 3))─┐ -│ (2,6) │ -└───────────────────────────────┘ -``` - -## tupleDivide {#tupledivide} - -Вычисляет частное соответствующих значений двух кортежей одинакового размера. Обратите внимание, что при делении на ноль возвращается значение `inf`. - -**Синтаксис** - -```sql -tupleDivide(tuple1, tuple2) -``` - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Кортеж с частными. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tupleDivide((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─tupleDivide((1, 2), (2, 3))─┐ -│ (0.5,0.6666666666666666) │ -└─────────────────────────────┘ -``` - -## tupleNegate {#tuplenegate} - -Применяет отрицание ко всем значениям кортежа. - -**Синтаксис** - -```sql -tupleNegate(tuple) -``` - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Кортеж с результатом отрицания. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tupleDivide((1, 2)); -``` - -Результат: - -```text -┌─tupleNegate((1, 2))─┐ -│ (-1,-2) │ -└─────────────────────┘ -``` - -## tupleMultiplyByNumber {#tuplemultiplybynumber} - -Возвращает кортеж, в котором все значения умножены на заданное число. - -**Синтаксис** - -```sql -tupleMultiplyByNumber(tuple, number) -``` - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `number` — множитель. [Int/UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Decimal](../../sql-reference/data-types/decimal.md). - -**Возвращаемое значение** - -- Кортеж с результатами умножения на число. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tupleMultiplyByNumber((1, 2), -2.1); -``` - -Результат: - -```text -┌─tupleMultiplyByNumber((1, 2), -2.1)─┐ -│ (-2.1,-4.2) │ -└─────────────────────────────────────┘ -``` - -## tupleDivideByNumber {#tupledividebynumber} - -Возвращает кортеж, в котором все значения поделены на заданное число. Обратите внимание, что при делении на ноль возвращается значение `inf`. - -**Синтаксис** - -```sql -tupleDivideByNumber(tuple, number) -``` - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `number` — делитель. [Int/UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). - -**Возвращаемое значение** - -- Кортеж с результатами деления на число. - -Тип: [Tuple](../../sql-reference/data-types/tuple.md). - -**Пример** - -Запрос: - -```sql -SELECT tupleDivideByNumber((1, 2), 0.5); -``` - -Результат: - -```text -┌─tupleDivideByNumber((1, 2), 0.5)─┐ -│ (2,4) │ -└──────────────────────────────────┘ -``` - -## dotProduct {#dotproduct} - -Вычисляет скалярное произведение двух кортежей одинакового размера. - -**Синтаксис** - -```sql -dotProduct(tuple1, tuple2) -``` - -Синоним: `scalarProduct`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - - -**Возвращаемое значение** - -- Скалярное произведение. - -Тип: [Int/UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Decimal](../../sql-reference/data-types/decimal.md). - -**Пример** - -Запрос: - -```sql -SELECT dotProduct((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─dotProduct((1, 2), (2, 3))─┐ -│ 8 │ -└────────────────────────────┘ -``` - -## L1Norm {#l1norm} - -Вычисляет сумму абсолютных значений кортежа. - -**Синтаксис** - -```sql -L1Norm(tuple) -``` - -Синоним: `normL1`. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- L1-норма или [расстояние городских кварталов](https://ru.wikipedia.org/wiki/%D0%A0%D0%B0%D1%81%D1%81%D1%82%D0%BE%D1%8F%D0%BD%D0%B8%D0%B5_%D0%B3%D0%BE%D1%80%D0%BE%D0%B4%D1%81%D0%BA%D0%B8%D1%85_%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB%D0%BE%D0%B2). - -Тип: [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) или [Decimal](../../sql-reference/data-types/decimal.md). - -**Пример** - -Запрос: - -```sql -SELECT L1Norm((1, 2)); -``` - -Результат: - -```text -┌─L1Norm((1, 2))─┐ -│ 3 │ -└────────────────┘ -``` - -## L2Norm {#l2norm} - -Вычисляет квадратный корень из суммы квадратов значений кортежа. - -**Синтаксис** - -```sql -L2Norm(tuple) -``` - -Синоним: `normL2`. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- L2-норма или [Евклидово расстояние](https://ru.wikipedia.org/wiki/%D0%95%D0%B2%D0%BA%D0%BB%D0%B8%D0%B4%D0%BE%D0%B2%D0%B0_%D0%BC%D0%B5%D1%82%D1%80%D0%B8%D0%BA%D0%B0). - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT L2Norm((1, 2)); -``` - -Результат: - -```text -┌───L2Norm((1, 2))─┐ -│ 2.23606797749979 │ -└──────────────────┘ -``` - -## LinfNorm {#linfnorm} - -Вычисляет максимум из абсолютных значений кортежа. - -**Синтаксис** - -```sql -LinfNorm(tuple) -``` - -Синоним: `normLinf`. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Linf-норма или максимальное абсолютное значение. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT LinfNorm((1, -2)); -``` - -Результат: - -```text -┌─LinfNorm((1, -2))─┐ -│ 2 │ -└───────────────────┘ -``` - -## LpNorm {#lpnorm} - -Возвращает корень степени `p` из суммы абсолютных значений кортежа, возведенных в степень `p`. - -**Синтаксис** - -```sql -LpNorm(tuple, p) -``` - -Синоним: `normLp`. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — степень. Возможные значение: любое число из промежутка [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). - -**Возвращаемое значение** - -- [Lp-норма](https://ru.wikipedia.org/wiki/%D0%9D%D0%BE%D1%80%D0%BC%D0%B0_(%D0%BC%D0%B0%D1%82%D0%B5%D0%BC%D0%B0%D1%82%D0%B8%D0%BA%D0%B0)#%D0%9D%D0%B5%D0%BA%D0%BE%D1%82%D0%BE%D1%80%D1%8B%D0%B5_%D0%B2%D0%B8%D0%B4%D1%8B_%D0%BC%D0%B0%D1%82%D1%80%D0%B8%D1%87%D0%BD%D1%8B%D1%85_%D0%BD%D0%BE%D1%80%D0%BC) - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT LpNorm((1, -2),2); -``` - -Результат: - -```text -┌─LpNorm((1, -2), 2)─┐ -│ 2.23606797749979 │ -└────────────────────┘ -``` - -## L1Distance {#l1distance} - -Вычисляет расстояние между двумя точками (значения кортежей — координаты точек) в норме L1 ([расстояние городских кварталов](https://ru.wikipedia.org/wiki/%D0%A0%D0%B0%D1%81%D1%81%D1%82%D0%BE%D1%8F%D0%BD%D0%B8%D0%B5_%D0%B3%D0%BE%D1%80%D0%BE%D0%B4%D1%81%D0%BA%D0%B8%D1%85_%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB%D0%BE%D0%B2)). - -**Синтаксис** - -```sql -L1Distance(tuple1, tuple2) -``` - -Синоним: `distanceL1`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Расстояние в норме L1. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT L1Distance((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─L1Distance((1, 2), (2, 3))─┐ -│ 2 │ -└────────────────────────────┘ -``` - -## L2Distance {#l2distance} - -Вычисляет расстояние между двумя точками (значения кортежей — координаты точек) в норме L2 ([Евклидово расстояние](https://ru.wikipedia.org/wiki/%D0%95%D0%B2%D0%BA%D0%BB%D0%B8%D0%B4%D0%BE%D0%B2%D0%B0_%D0%BC%D0%B5%D1%82%D1%80%D0%B8%D0%BA%D0%B0)). - -**Синтаксис** - -```sql -L2Distance(tuple1, tuple2) -``` - -Синоним: `distanceL2`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Расстояние в норме L2. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT L2Distance((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─L2Distance((1, 2), (2, 3))─┐ -│ 1.4142135623730951 │ -└────────────────────────────┘ -``` - -## LinfDistance {#linfdistance} - -Вычисляет расстояние между двумя точками (значения кортежей — координаты точек) в норме [Linf](https://ru.wikipedia.org/wiki/%D0%9D%D0%BE%D1%80%D0%BC%D0%B0_(%D0%BC%D0%B0%D1%82%D0%B5%D0%BC%D0%B0%D1%82%D0%B8%D0%BA%D0%B0)#%D0%9D%D0%B5%D0%BA%D0%BE%D1%82%D0%BE%D1%80%D1%8B%D0%B5_%D0%B2%D0%B8%D0%B4%D1%8B_%D0%BC%D0%B0%D1%82%D1%80%D0%B8%D1%87%D0%BD%D1%8B%D1%85_%D0%BD%D0%BE%D1%80%D0%BC). - -**Синтаксис** - -```sql -LinfDistance(tuple1, tuple2) -``` - -Синоним: `distanceLinf`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемые значения** - -- Расстояние в норме Linf. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT LinfDistance((1, 2), (2, 3)); -``` - -Результат: - -```text -┌─LinfDistance((1, 2), (2, 3))─┐ -│ 1 │ -└──────────────────────────────┘ -``` - -## LpDistance {#lpdistance} - -Вычисляет расстояние между двумя точками (значения кортежей — координаты точек) в норме [Lp](https://ru.wikipedia.org/wiki/%D0%9D%D0%BE%D1%80%D0%BC%D0%B0_(%D0%BC%D0%B0%D1%82%D0%B5%D0%BC%D0%B0%D1%82%D0%B8%D0%BA%D0%B0)#%D0%9D%D0%B5%D0%BA%D0%BE%D1%82%D0%BE%D1%80%D1%8B%D0%B5_%D0%B2%D0%B8%D0%B4%D1%8B_%D0%BC%D0%B0%D1%82%D1%80%D0%B8%D1%87%D0%BD%D1%8B%D1%85_%D0%BD%D0%BE%D1%80%D0%BC). - -**Синтаксис** - -```sql -LpDistance(tuple1, tuple2, p) -``` - -Синоним: `distanceLp`. - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — степень. Возможные значение: любое число из промежутка [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). - -**Возвращаемое значение** - -- Расстояние в норме Lp. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT LpDistance((1, 2), (2, 3), 3); -``` - -Результат: - -```text -┌─LpDistance((1, 2), (2, 3), 3)─┐ -│ 1.2599210498948732 │ -└───────────────────────────────┘ -``` - -## L1Normalize {#l1normalize} - -Вычисляет единичный вектор для исходного вектора (значения кортежа — координаты вектора) в норме L1 ([расстояние городских кварталов](https://ru.wikipedia.org/wiki/%D0%A0%D0%B0%D1%81%D1%81%D1%82%D0%BE%D1%8F%D0%BD%D0%B8%D0%B5_%D0%B3%D0%BE%D1%80%D0%BE%D0%B4%D1%81%D0%BA%D0%B8%D1%85_%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB%D0%BE%D0%B2)). - -**Синтаксис** - -```sql -L1Normalize(tuple) -``` - -Синоним: `normalizeL1`. - -**Аргументы** - -- `tuple` — [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Единичный вектор. - -Тип: кортеж [Tuple](../../sql-reference/data-types/tuple.md) значений [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT L1Normalize((1, 2)); -``` - -Результат: - -```text -┌─L1Normalize((1, 2))─────────────────────┐ -│ (0.3333333333333333,0.6666666666666666) │ -└─────────────────────────────────────────┘ -``` - -## L2Normalize {#l2normalize} - -Вычисляет единичный вектор для исходного вектора (значения кортежа — координаты вектора) в норме L2 ([Евклидово пространство](https://ru.wikipedia.org/wiki/%D0%95%D0%B2%D0%BA%D0%BB%D0%B8%D0%B4%D0%BE%D0%B2%D0%BE_%D0%BF%D1%80%D0%BE%D1%81%D1%82%D1%80%D0%B0%D0%BD%D1%81%D1%82%D0%B2%D0%BE). - -**Синтаксис** - -```sql -L2Normalize(tuple) -``` - -Синоним: `normalizeL1`. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Единичный вектор. - -Тип: кортеж [Tuple](../../sql-reference/data-types/tuple.md) значений [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT L2Normalize((3, 4)); -``` - -Результат: - -```text -┌─L2Normalize((3, 4))─┐ -│ (0.6,0.8) │ -└─────────────────────┘ -``` - -## LinfNormalize {#linfnormalize} - -Вычисляет единичный вектор для исходного вектора (значения кортежа — координаты вектора) в норме [Linf](https://ru.wikipedia.org/wiki/%D0%9D%D0%BE%D1%80%D0%BC%D0%B0_(%D0%BC%D0%B0%D1%82%D0%B5%D0%BC%D0%B0%D1%82%D0%B8%D0%BA%D0%B0)#%D0%9D%D0%B5%D0%BA%D0%BE%D1%82%D0%BE%D1%80%D1%8B%D0%B5_%D0%B2%D0%B8%D0%B4%D1%8B_%D0%BC%D0%B0%D1%82%D1%80%D0%B8%D1%87%D0%BD%D1%8B%D1%85_%D0%BD%D0%BE%D1%80%D0%BC). - -**Синтаксис** - -```sql -LinfNormalize(tuple) -``` - -Синоним: `normalizeLinf `. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемое значение** - -- Единичный вектор. - -Тип: кортеж [Tuple](../../sql-reference/data-types/tuple.md) значений [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT LinfNormalize((3, 4)); -``` - -Результат: - -```text -┌─LinfNormalize((3, 4))─┐ -│ (0.75,1) │ -└───────────────────────┘ -``` - -## LpNormalize {#lpnormalize} - -Вычисляет единичный вектор для исходного вектора (значения кортежа — координаты вектора) в норме [Lp](https://ru.wikipedia.org/wiki/%D0%9D%D0%BE%D1%80%D0%BC%D0%B0_(%D0%BC%D0%B0%D1%82%D0%B5%D0%BC%D0%B0%D1%82%D0%B8%D0%BA%D0%B0)#%D0%9D%D0%B5%D0%BA%D0%BE%D1%82%D0%BE%D1%80%D1%8B%D0%B5_%D0%B2%D0%B8%D0%B4%D1%8B_%D0%BC%D0%B0%D1%82%D1%80%D0%B8%D1%87%D0%BD%D1%8B%D1%85_%D0%BD%D0%BE%D1%80%D0%BC). - -**Синтаксис** - -```sql -LpNormalize(tuple, p) -``` - -Синоним: `normalizeLp `. - -**Аргументы** - -- `tuple` — кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `p` — степень. Возможные значение: любое число из промежутка [1;inf). [UInt](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). - -**Возвращаемое значение** - -- Единичный вектор. - -Тип: кортеж [Tuple](../../sql-reference/data-types/tuple.md) значений [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT LpNormalize((3, 4),5); -``` - -Результат: - -```text -┌─LpNormalize((3, 4), 5)──────────────────┐ -│ (0.7187302630182624,0.9583070173576831) │ -└─────────────────────────────────────────┘ -``` - -## cosineDistance {#cosinedistance} - -Вычисляет косинусную разницу двух векторов (значения кортежей — координаты векторов). Чем меньше возвращаемое значение, тем больше сходство между векторами. - -**Синтаксис** - -```sql -cosineDistance(tuple1, tuple2) -``` - -**Аргументы** - -- `tuple1` — первый кортеж. [Tuple](../../sql-reference/data-types/tuple.md). -- `tuple2` — второй кортеж. [Tuple](../../sql-reference/data-types/tuple.md). - -**Возвращаемые значения** - -- Результат разности между единицей и косинуса угла между векторами. - -Тип: [Float](../../sql-reference/data-types/float.md). - -**Пример** - -Запрос: - -```sql -SELECT cosineDistance((1, 2),(2,3)); -``` - -Результат: - -```text -┌─cosineDistance((1, 2), (2, 3))─┐ -│ 0.007722123286332261 │ -└────────────────────────────────┘ -``` diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 08054c28ab8..98f6873f712 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -18,30 +18,20 @@ toc_title: "Операторы" `-a` - функция `negate(a)`. -Для отрицания чисел в кортеже: [tupleNegate](../../sql-reference/functions/tuple-functions.md#tuplenegate). - ## Операторы умножения и деления {#operatory-umnozheniia-i-deleniia} `a * b` - функция `multiply(a, b)` -Для умножения кортежа на число: [tupleMultiplyByNumber](../../sql-reference/functions/tuple-functions.md#tuplemultiplybynumber), для скалярного произведения: [dotProduct](../../sql-reference/functions/tuple-functions.md#dotproduct). - `a / b` - функция `divide(a, b)` -Для деления кортежа на число: [tupleDivideByNumber](../../sql-reference/functions/tuple-functions.md#tupledividebynumber). - `a % b` - функция `modulo(a, b)` ## Операторы сложения и вычитания {#operatory-slozheniia-i-vychitaniia} `a + b` - функция `plus(a, b)` -Для сложения кортежей: [tuplePlus](../../sql-reference/functions/tuple-functions.md#tupleplus). - `a - b` - функция `minus(a, b)` -Для вычитания кортежей: [tupleMinus](../../sql-reference/functions/tuple-functions.md#tupleminus). - ## Операторы сравнения {#operatory-sravneniia} `a = b` - функция `equals(a, b)` From 8e680b068ad4d57db3701a953943451959a8db76 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Mon, 25 Oct 2021 16:53:23 +0300 Subject: [PATCH 071/173] Add comment with fix explanation --- src/Common/ProgressIndication.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 5a3f8cfc350..db1daaac415 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -122,6 +122,9 @@ ProgressIndication::MemoryUsage ProgressIndication::getMemoryUsage() const [](MemoryUsage const & acc, auto const & host_data) { UInt64 host_usage = 0; + // In ProfileEvents packets thread id 0 specifies common profiling information + // for all threads executing current query on specific host. So instead of summing per thread + // memory consumption it's enough to look for data with thread id 0. if (auto it = host_data.second.find(ZERO); it != host_data.second.end()) host_usage = it->second.memory_usage; return MemoryUsage{.total = acc.total + host_usage, .max = std::max(acc.max, host_usage)}; From 94088f8197f0bd530e04362c5132bf2d84d20d8c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 25 Oct 2021 17:01:23 +0300 Subject: [PATCH 072/173] minor fixes --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 9 +++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 3 +- src/Storages/StorageReplicatedMergeTree.cpp | 37 +++++++++++++++---- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 6c1cd04d4ae..985d54e8aee 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1004,7 +1004,7 @@ bool ReplicatedMergeTreeQueue::checkReplaceRangeCanBeRemoved(const MergeTreePart void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( zkutil::ZooKeeperPtr zookeeper, const MergeTreePartInfo & part_info, - const std::optional & current) + const std::optional & covering_entry) { /// TODO is it possible to simplify it? Queue to_wait; @@ -1015,8 +1015,9 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( /// Remove operations with parts, contained in the range to be deleted, from the queue. std::unique_lock lock(state_mutex); - [[maybe_unused]] bool called_from_alter_query_directly = current && current->replace_range_entry && current->replace_range_entry->columns_version < 0; - [[maybe_unused]] bool called_for_broken_part = !current; + [[maybe_unused]] bool called_from_alter_query_directly = covering_entry && covering_entry->replace_range_entry + && covering_entry->replace_range_entry->columns_version < 0; + [[maybe_unused]] bool called_for_broken_part = !covering_entry; assert(currently_executing_drop_or_replace_range || called_from_alter_query_directly || called_for_broken_part); for (Queue::iterator it = queue.begin(); it != queue.end();) @@ -1028,7 +1029,7 @@ void ReplicatedMergeTreeQueue::removePartProducingOpsInRange( type == LogEntry::MUTATE_PART; bool simple_op_covered = is_simple_producing_op && part_info.contains(MergeTreePartInfo::fromPartName((*it)->new_part_name, format_version)); - bool replace_range_covered = current && checkReplaceRangeCanBeRemoved(part_info, *it, *current); + bool replace_range_covered = covering_entry && checkReplaceRangeCanBeRemoved(part_info, *it, *covering_entry); if (simple_op_covered || replace_range_covered) { if ((*it)->currently_executing) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index c059ff7a072..273ac21eebc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -325,9 +325,10 @@ public: /** Remove the action from the queue with the parts covered by part_name (from ZK and from the RAM). * And also wait for the completion of their execution, if they are now being executed. + * covering_entry is as an entry that caused removal of entries in range (usually, DROP_RANGE) */ void removePartProducingOpsInRange(zkutil::ZooKeeperPtr zookeeper, const MergeTreePartInfo & part_info, - const std::optional & current); + const std::optional & covering_entry); /** In the case where there are not enough parts to perform the merge in part_name * - move actions with merged parts to the end of the queue diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 052303e3627..f7bccb7e823 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2319,6 +2319,13 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo LogEntryPtr parsed_entry = {}; }; + /// We got log pointer and list of queue entries of source replica. + /// At first we will get queue entries and then we will get list of active parts of source replica + /// to enqueue fetches for missing parts. If source replica executes and removes some entry concurrently + /// we will see produced part (or covering part) in replicas/source/parts and will enqueue fetch. + /// We will try to parse queue entries before copying them + /// to avoid creation of excessive and duplicating entries in our queue. + /// See also removePartAndEnqueueFetch(...) std::vector source_queue; ActiveDataPartSet get_part_set{format_version}; ActiveDataPartSet drop_range_set{format_version}; @@ -2358,11 +2365,15 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo info.parsed_entry->znode_name = source_queue_names[i]; - if (info.parsed_entry->type == LogEntry::GET_PART) - get_part_set.add(info.parsed_entry->new_part_name); - if (info.parsed_entry->type == LogEntry::DROP_RANGE) drop_range_set.add(info.parsed_entry->new_part_name); + + if (info.parsed_entry->type == LogEntry::GET_PART) + { + String maybe_covering_drop_range = drop_range_set.getContainingPart(info.parsed_entry->new_part_name); + if (maybe_covering_drop_range.empty()) + get_part_set.add(info.parsed_entry->new_part_name); + } } } @@ -2396,7 +2407,8 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo Coordination::Stat is_lost_stat_new; zookeeper->get(fs::path(source_path) / "is_lost", &is_lost_stat_new); if (is_lost_stat_new.version != source_is_lost_stat.version) - throw Exception(ErrorCodes::REPLICA_STATUS_CHANGED, "Cannot clone {}, because it suddenly become lost", source_replica); + throw Exception(ErrorCodes::REPLICA_STATUS_CHANGED, "Cannot clone {}, because it suddenly become lost " + "or removed broken part from ZooKeeper", source_replica); } tryRemovePartsFromZooKeeperWithRetries(parts_to_remove_from_zk); @@ -2431,7 +2443,8 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo /// Avoid creation of GET_PART entries which covered by another GET_PART or DROP_RANGE /// and creation of multiple entries with the same new_part_name. - auto should_ignore_log_entry = [&] (const String & part_name, const String & log_msg_context) -> bool + auto should_ignore_log_entry = [&drop_range_set, &get_part_set, this] (std::unordered_set & created_gets, + const String & part_name, const String & log_msg_context) -> bool { /// We should not create entries covered by DROP_RANGE, because we will remove them anyway (kind of optimization). String covering_drop_range = drop_range_set.getContainingPart(part_name); @@ -2460,7 +2473,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo /// but before we copied its active parts set. In this case we will GET_PART entry in our queue /// and later will pull the original GET_PART from replication log. /// It should not cause any issues, but it does not allow to get rid of duplicated entries and add an assertion. - if (created_get_parts.count(part_name)) + if (created_gets.count(part_name)) { /// NOTE It would be better to copy log entry instead of creating GET_PART /// if there are GET_PART and log entry of other type with the same new_part_name. @@ -2474,7 +2487,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo for (const String & name : active_parts) { - if (should_ignore_log_entry(name, "Not fetching")) + if (should_ignore_log_entry(created_get_parts, name, "Not fetching")) continue; LogEntry log_entry; @@ -2532,7 +2545,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo const String & entry_name = entry_info.parsed_entry->znode_name; const auto & entry_type = entry_info.parsed_entry->type; - if (should_ignore_log_entry(part_name, fmt::format("Not copying {} {} ", entry_name, entry_type))) + if (should_ignore_log_entry(created_get_parts, part_name, fmt::format("Not copying {} {} ", entry_name, entry_type))) continue; if (entry_info.parsed_entry->type == LogEntry::GET_PART) @@ -3317,6 +3330,14 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n Coordination::Stat stat; if (zookeeper->exists(part_path, &stat)) { + /// Update version of /is_lost node to avoid race condition with cloneReplica(...). + /// cloneReplica(...) expects that if some entry was executed, then its new_part_name is added to /parts, + /// but we are going to remove it from /parts and add to queue again. + Coordination::Stat is_lost_stat; + String is_lost_value = zookeeper->get(replica_path + "/is_lost", &is_lost_stat); + assert(is_lost_value == "0"); + ops.emplace_back(zkutil::makeSetRequest(replica_path + "/is_lost", is_lost_value, is_lost_stat.version)); + part_create_time = stat.ctime / 1000; removePartFromZooKeeper(part_name, ops, stat.numChildren > 0); } From 55e887728e8736e14c4d35b6cced0c38c780f62f Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 25 Oct 2021 12:35:09 +0300 Subject: [PATCH 073/173] Retry on Yandex.S3 throttle --- contrib/aws | 2 +- .../configs/config.d/storage_conf.xml | 16 ++++++++++ .../s3_endpoint/endpoint.py | 19 ++++++++++++ .../test_merge_tree_s3_failover/test.py | 30 +++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/contrib/aws b/contrib/aws index 06aa8759d17..00b03604543 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 06aa8759d17f2032ffd5efa83969270ca9ac727b +Subproject commit 00b03604543367d7e310cb0993973fdcb723ea79 diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index f5e350e5b6b..32d78468a71 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -12,6 +12,15 @@ 0 + + s3 + + http://resolver:8080/root/data/ + minio + minio123 + + true + @@ -32,6 +41,13 @@ + + +
+ s3_retryable +
+ + diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index 1754b1f175c..3f219b6ba57 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -18,6 +18,16 @@ def fail_request(_request_number): return 'OK' +@route('/throttle_request/<_request_number>') +def fail_request(_request_number): + request_number = int(_request_number) + if request_number > 0: + cache['throttle_request_number'] = request_number + else: + cache.pop('throttle_request_number', None) + return 'OK' + + # Handle for MultipleObjectsDelete. @route('/<_bucket>', ['POST']) def delete(_bucket): @@ -37,6 +47,15 @@ def server(_bucket, _path): response.content_type = 'text/xml' return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' + if cache.get('throttle_request_number', None): + request_number = cache.pop('throttle_request_number') - 1 + if request_number > 0: + cache['throttle_request_number'] = request_number + else: + response.status = 429 + response.content_type = 'text/xml' + return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' + response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) response.status = 307 return 'Redirected' diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index 56d9441aba6..b6b47417523 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -38,6 +38,12 @@ def fail_request(cluster, request): assert response == 'OK', 'Expected "OK", but got "{}"'.format(response) +def throttle_request(cluster, request): + response = cluster.exec_in_container(cluster.get_container_id('resolver'), + ["curl", "-s", "http://resolver:8080/throttle_request/{}".format(request)]) + assert response == 'OK', 'Expected "OK", but got "{}"'.format(response) + + @pytest.fixture(scope="module") def cluster(): try: @@ -186,3 +192,27 @@ def test_move_failover(cluster): # Ensure data is not corrupted. assert node.query("CHECK TABLE s3_failover_test") == '1\n' assert node.query("SELECT id,data FROM s3_failover_test FORMAT Values") == "(0,'data'),(1,'data')" + + +# Check that throttled request retries and does not cause an error on disk with default `retry_attempts` (>0) +def test_throttle_retry(cluster): + node = cluster.instances["node"] + + node.query( + """ + CREATE TABLE s3_throttle_retry_test ( + id Int64 + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS storage_policy='s3_retryable' + """ + ) + + data = "(42)" + node.query("INSERT INTO s3_throttle_retry_test VALUES {}".format(data)) + + throttle_request(cluster, 1) + + assert node.query(""" + SELECT * FROM s3_throttle_retry_test + """) == '42\n' From 5cdd9a4b99c84f44e105c064a955356c02a9749a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 25 Oct 2021 19:00:55 +0300 Subject: [PATCH 074/173] fix intersecting parts in parts_to_do --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f397e73aa31..29a13970b3e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -203,9 +203,15 @@ void ReplicatedMergeTreeQueue::insertUnlocked( drop_ranges.addDropRange(entry); /// DROP PART remove parts, so we remove it from virtual parts to - /// preserve invariant virtual_parts = current_parts + queue + /// preserve invariant virtual_parts = current_parts + queue. + /// Also remove it from parts_to_do to avoid intersecting parts in parts_to_do + /// if fast replica will execute DROP PART and assign a merge that contains dropped blocks. if (entry->isDropPart(format_version)) - virtual_parts.removePartAndCoveredParts(*entry->getDropRange(format_version)); + { + String drop_part_name = *entry->getDropRange(format_version); + virtual_parts.removePartAndCoveredParts(drop_part_name); + removeCoveredPartsFromMutations(drop_part_name, /*remove_part = */ true, /*remove_covered_parts = */ true); + } queue.push_front(entry); } From 5231e8b8d9dfdea1f95e4e78a87c46b1d870e256 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 25 Oct 2021 20:12:29 +0300 Subject: [PATCH 075/173] HashedDictionaries added read performance tests --- tests/performance/hashed_array_dictionary.xml | 10 ++++++++++ tests/performance/hashed_dictionary.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/performance/hashed_array_dictionary.xml b/tests/performance/hashed_array_dictionary.xml index c992be117d3..80a565e67b0 100644 --- a/tests/performance/hashed_array_dictionary.xml +++ b/tests/performance/hashed_array_dictionary.xml @@ -103,6 +103,11 @@ FORMAT Null; + + SELECT * FROM simple_key_hashed_array_dictionary + FORMAT Null; + + WITH (rand64() % toUInt64({elements_count}), toString(rand64() % toUInt64({elements_count}))) as key SELECT dictGet('default.complex_key_hashed_array_dictionary', {column_name}, key) @@ -119,6 +124,11 @@ FORMAT Null; + + SELECT * FROM complex_key_hashed_array_dictionary + FORMAT Null; + + DROP TABLE IF EXISTS simple_key_hashed_array_dictionary_source_table; DROP TABLE IF EXISTS complex_key_hashed_array_dictionary_source_table; diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index fcc8c1bd539..cf1cdac6df1 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -103,6 +103,11 @@ FORMAT Null; + + SELECT * FROM simple_key_hashed_dictionary + FORMAT Null; + + WITH (rand64() % toUInt64({elements_count}), toString(rand64() % toUInt64({elements_count}))) as key SELECT dictGet('default.complex_key_hashed_dictionary', {column_name}, key) @@ -119,6 +124,11 @@ FORMAT Null; + + SELECT * FROM complex_key_hashed_dictionary + FORMAT Null; + + DROP TABLE IF EXISTS simple_key_hashed_dictionary_source_table; DROP TABLE IF EXISTS complex_key_hashed_dictionary_source_table; From 7fe028245ea9e395dc16b7c1adf07b2ea52fa479 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 25 Oct 2021 21:12:45 +0300 Subject: [PATCH 076/173] Do not save preprocessed config for clickhouse-local It seems that there is no purpose on doing this, and it just produce an error due to permission errors (i.e. before this patch if you will pass small config with some overrides then it will try to write to /var/lib/clickhouse by default). --- programs/local/LocalServer.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 69f7820909a..b45bb991887 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -179,7 +179,6 @@ void LocalServer::initialize(Poco::Util::Application & self) ConfigProcessor config_processor(config_path, false, true); config_processor.setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", ".")); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } @@ -380,7 +379,6 @@ void LocalServer::setupUsers() const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); ConfigProcessor config_processor(users_config_path); const auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH)); users_config = loaded_config.configuration; } else From 096eaa1008c8104da37a147ece75831b7ca7309a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 25 Oct 2021 21:12:45 +0300 Subject: [PATCH 077/173] Initialize custom TLDs in clickhouse-local --- programs/local/LocalServer.cpp | 9 +++++++- ...110_clickhouse_local_custom_tld.config.xml | 22 +++++++++++++++++++ ...2110_clickhouse_local_custom_tld.reference | 1 + .../02110_clickhouse_local_custom_tld.sh | 21 ++++++++++++++++++ .../02110_clickhouse_local_custom_tld.tld.dat | 1 + 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02110_clickhouse_local_custom_tld.config.xml create mode 100644 tests/queries/0_stateless/02110_clickhouse_local_custom_tld.reference create mode 100755 tests/queries/0_stateless/02110_clickhouse_local_custom_tld.sh create mode 100644 tests/queries/0_stateless/02110_clickhouse_local_custom_tld.tld.dat diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b45bb991887..2a2fe4a78c8 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -18,7 +18,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -35,7 +37,6 @@ #include #include #include -#include #include namespace fs = std::filesystem; @@ -283,6 +284,11 @@ void LocalServer::tryInitPath() global_context->setFlagsPath(path + "flags"); global_context->setUserFilesPath(""); // user's files are everywhere + + /// top_level_domains_lists + const std::string & top_level_domains_path = config().getString("top_level_domains_path", path + "top_level_domains/"); + if (!top_level_domains_path.empty()) + TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); } @@ -671,6 +677,7 @@ void LocalServer::addOptions(OptionsDescription & options_description) ("no-system-tables", "do not attach system tables (better startup time)") ("path", po::value(), "Storage path") + ("top_level_domains_path", po::value(), "Path to lists with custom TLDs") ; } diff --git a/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.config.xml b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.config.xml new file mode 100644 index 00000000000..0baa5b32ef8 --- /dev/null +++ b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.config.xml @@ -0,0 +1,22 @@ + + + + + + + + + ::/0 + + default + default + + + + + + + + 02110_clickhouse_local_custom_tld.tld.dat + + diff --git a/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.reference b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.reference new file mode 100644 index 00000000000..bf62392aeee --- /dev/null +++ b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.reference @@ -0,0 +1 @@ +biz.ss kernel.biz.ss diff --git a/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.sh b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.sh new file mode 100755 index 00000000000..02a454957d4 --- /dev/null +++ b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +function clickhouse_local() +{ + local opts=( + --config "$CURDIR/$(basename "${BASH_SOURCE[0]}" .sh).config.xml" + --top_level_domains_path "$CURDIR" + ) + $CLICKHOUSE_LOCAL "${opts[@]}" "$@" +} + +# -- biz.ss is not in the default TLD list, hence: +clickhouse_local -q " + select + cutToFirstSignificantSubdomain('foo.kernel.biz.ss'), + cutToFirstSignificantSubdomainCustom('foo.kernel.biz.ss', 'public_suffix_list') +" |& grep -v -e 'Processing configuration file' diff --git a/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.tld.dat b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.tld.dat new file mode 100644 index 00000000000..510e6dd9ec0 --- /dev/null +++ b/tests/queries/0_stateless/02110_clickhouse_local_custom_tld.tld.dat @@ -0,0 +1 @@ +biz.ss From f41cf281e4a29e9d703bde1e29c74a7433f1a208 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 25 Oct 2021 21:14:16 +0300 Subject: [PATCH 078/173] Add QueryProfilerRuns profile event This will help to investigate query latency spikes. --- src/Common/ProfileEvents.cpp | 3 ++- src/Common/QueryProfiler.cpp | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 941a3ab0896..44adbccc089 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -227,7 +227,8 @@ M(CreatedHTTPConnections, "Total amount of created HTTP connections (counter increase every time connection is created).") \ \ M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \ - M(QueryProfilerSignalOverruns, "Number of times we drop processing of a signal due to overrun plus the number of signals that OS has not delivered due to overrun.") \ + M(QueryProfilerSignalOverruns, "Number of times we drop processing of a query profiler signal due to overrun plus the number of signals that OS has not delivered due to overrun.") \ + M(QueryProfilerRuns, "Number of times QueryProfiler had been run.") \ \ M(CreatedLogEntryForMerge, "Successfully created log entry to merge parts in ReplicatedMergeTree.") \ M(NotCreatedLogEntryForMerge, "Log entry to merge parts in ReplicatedMergeTree is not created due to concurrent log update by another replica.") \ diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index aa40226093a..77b8d0dda5e 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -15,6 +15,7 @@ namespace ProfileEvents { extern const Event QueryProfilerSignalOverruns; + extern const Event QueryProfilerRuns; } namespace DB @@ -60,6 +61,7 @@ namespace const StackTrace stack_trace(signal_context); TraceCollector::collect(trace_type, stack_trace, 0); + ProfileEvents::increment(ProfileEvents::QueryProfilerRuns); errno = saved_errno; } From 0b7a8c98871a704e9d093be006bef4d6dfce961a Mon Sep 17 00:00:00 2001 From: gyuton Date: Mon, 25 Oct 2021 22:35:14 +0300 Subject: [PATCH 079/173] minor fix --- docs/en/operations/settings/settings.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 3eb39907991..47e0a3006da 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3834,7 +3834,7 @@ Default value: `0`. ## async_insert_threads {#async-insert-threads} -The maximum number of threads for backgroung data parsing and insertion. +The maximum number of threads for background data parsing and insertion. Possible values: @@ -3858,6 +3858,8 @@ Default value: `1`. The timeout in seconds for waiting for processing of asynchronous insertion. +Possible values: + - Positive integer. - 0 — Disabled. @@ -3878,6 +3880,8 @@ Default value: `1000000`. The maximum timeout in milliseconds since the first `INSERT` query before inserting collected data. +Possible values: + - Positive integer. - 0 — Timeout disabled. @@ -3887,6 +3891,8 @@ Default value: `200`. The maximum timeout in milliseconds since the last `INSERT` query before dumping collected data. If enabled, the settings prolongs the [async_insert_busy_timeout](#async-insert-busy-timeout) with every `INSERT` query as long as [async_insert_max_data_size](#async-insert-max-data-size) is not exceeded. +Possible values: + - Positive integer. - 0 — Timeout disabled. From c08f6cae0523e32b49c6a3cb49bb79f8668b4c26 Mon Sep 17 00:00:00 2001 From: gyuton Date: Mon, 25 Oct 2021 22:39:47 +0300 Subject: [PATCH 080/173] translated to Russian --- .../sql-reference/statements/insert-into.md | 10 +++ docs/ru/operations/settings/settings.md | 84 +++++++++++++++++++ .../sql-reference/statements/insert-into.md | 11 +++ 3 files changed, 105 insertions(+) diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index c44f7435089..ad5ab705937 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -120,3 +120,13 @@ Performance will not decrease if: - You upload data that is usually sorted by time. It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections. The data from such insertions is combined into batches and then safely inserted into a table. To enable the asynchronous mode, switch on the [async_insert](../../operations/settings/settings.md#async-insert) setting. Note that asynchronous insertions are supported only over HTTP protocol, and deduplication is not supported for them. + +**See Also** + +- [async_insert](../../operations/settings/settings.md#async-insert) +- [async_insert_threads](../../operations/settings/settings.md#async-insert-threads) +- [wait_for_async_insert](../../operations/settings/settings.md#wait-for-async-insert) +- [wait_for_async_insert_timeout](../../operations/settings/settings.md#wait-for-async-insert-timeout) +- [async_insert_max_data_size](../../operations/settings/settings.md#async-insert-max-data-size) +- [async_insert_busy_timeout](../../operations/settings/settings.md#async-insert-busy-timeout) +- [async_insert_stale_timeout](../../operations/settings/settings.md#async-insert-stale-timeout) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 71b6ac513fc..fb84913c0db 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3640,3 +3640,87 @@ SELECT * FROM positional_arguments ORDER BY 2,3; **См. также** - настройка [optimize_move_to_prewhere](#optimize_move_to_prewhere) + +## async_insert {#async-insert} + +Включает или отключает асинхронные вставки для нескольких параллельных соединений. Работает только для вставок по протоколу HTTP. Обратите внимание, что при таких вставках дедупликация не производится. + +Если включено, данные собираются в пачки перед вставкой в таблицу. Это позволяет производить мелкие и частые вставки в Clickhouse (до 15000 запросов в секунду) без промежуточных таблиц. + +Вставка данных происходит либо как только привышен [async_insert_max_data_size](#async-insert-max-data-size), либо через [async_insert_busy_timeout](#async-insert-busy-timeout) миллисекунд после первого запроса `INSERT`. Если [async_insert_stale_timeout](#async-insert-stale-timeout) задано ненулевое значение, то данные вставляются через `async_insert_stale_timeout` после последнего запроса. + +Если включен параметр [wait_for_async_insert](#wait-for-async-insert), каждый клиент ждет, пока данные будут сброшены в таблицу. Иначе запрос будет обработан почти моментально, даже если данные еще не вставлены. + +Возможные значения: + +- 0 — вставки производятся синхронно, один запрос за другим. +- 1 — включены множественные асинхронные вставки. + +Значение по умолчанию: `0`. + +## async_insert_threads {#async-insert-threads} + +Максимальное число потоков для фоновой обработки и вставки данных. + +Возможные значения: + +- положительное целое число. +- 0 — асинхронные вставки отключены. + +Значение по умолчанию: `16`. + +## wait_for_async_insert {#wait-for-async-insert} + +Включает или отключает ожидание обработки асинхронных вставок. Если включено, клиент выведет `OK` только после того, как данные вставлены. Иначе будет выведен `OK`, даже если вставка не произошла. + +Возможные значения: + +- 0 — отключено. +- 1 — включено. + +Значение по умолчанию: `1`. + +## wait_for_async_insert_timeout {#wait-for-async-insert-timeout} + +Время ожидания в секундах, выделяемое для обработки асинхронной вставки. + +Возможные значения: + +- Положительное целое число. +- 0 — ожидание отключено. + +Значение по умолчанию: [lock_acquire_timeout](#lock_acquire_timeout). + +## async_insert_max_data_size {#async-insert-max-data-size} + +Максимальный размер необработанных данных (в байтах), собранных за запрос, перед их вставкой. + +Возможные значения: + +- Положительное целое число. +- 0 — асинхронные вставки отключены. + +Значение по умолчанию: `1000000`. + +## async_insert_busy_timeout {#async-insert-busy-timeout} + +Максимальное время ожидания в миллисекундах после первого запроса `INSERT` и перед вставкой данных. + +Возможные значения: + +- Положительное целое число. +- 0 — ожидание отключено. + +Значение по умолчанию: `200`. + +## async_insert_stale_timeout {#async-insert-stale-timeout} + +Максимальное время ожидания в миллисекундах после последнего запроса `INSERT` и перед вставкой данных. Если установлено ненулевое значение, [async_insert_busy_timeout](#async-insert-busy-timeout) будет продлеваться с каждым запросом `INSERT`, пока не будет превышен [async_insert_max_data_size](#async-insert-max-data-size). + +Возможные значения: + +- Положительное целое число. +- 0 — ожидание отключено. + +Значение по умолчанию: `0`. + diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index da9f3a11101..d7732cc58e1 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -121,3 +121,14 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - Данные поступают в режиме реального времени. - Вы загружаете данные, которые как правило отсортированы по времени. +Также возможно вставлять данные асинхронно во множественных маленьких вставках от нескольких параллельных соединений. Данные от таких вставок сначала собираются в пачки, а потом вставляются в таблицу. Чтобы включить асинхронный режим, используйте настройку [async_insert](../../operations/settings/settings.md#async-insert). Обратите внимание, что асинхронные вставки поддерживаются только через протокол HTTP, а дедупликация при этом не производится. + +**См. также** + +- [async_insert](../../operations/settings/settings.md#async-insert) +- [async_insert_threads](../../operations/settings/settings.md#async-insert-threads) +- [wait_for_async_insert](../../operations/settings/settings.md#wait-for-async-insert) +- [wait_for_async_insert_timeout](../../operations/settings/settings.md#wait-for-async-insert-timeout) +- [async_insert_max_data_size](../../operations/settings/settings.md#async-insert-max-data-size) +- [async_insert_busy_timeout](../../operations/settings/settings.md#async-insert-busy-timeout) +- [async_insert_stale_timeout](../../operations/settings/settings.md#async-insert-stale-timeout) From 3d115536f53ac8dc9af4b508ccc6cbc4d2ea569e Mon Sep 17 00:00:00 2001 From: jasperzhu Date: Tue, 26 Oct 2021 01:39:09 +0530 Subject: [PATCH 081/173] Improve filter bitmask generator function all in one with sse/avx2/avx512 --- src/CMakeLists.txt | 1 + src/Columns/ColumnDecimal.cpp | 38 ++++++++++----- src/Columns/ColumnFixedString.cpp | 77 ++----------------------------- src/Columns/ColumnVector.cpp | 72 ++--------------------------- src/Columns/ColumnsCommon.cpp | 61 +++++++++++++++--------- src/Columns/ColumnsCommon.h | 2 + 6 files changed, 76 insertions(+), 175 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e8c3703672..9e1f25c21d9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -294,6 +294,7 @@ set_source_files_properties( Columns/ColumnFixedString.cpp Columns/ColumnsCommon.cpp Columns/ColumnVector.cpp + Columns/ColumnDecimal.cpp PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}") if(RE2_LIBRARY) diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index dc236fafbd9..48f890edaaf 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -237,25 +237,39 @@ ColumnPtr ColumnDecimal::filter(const IColumn::Filter & filt, ssize_t result_ const UInt8 * filt_end = filt_pos + size; const T * data_pos = data.data(); -#ifdef __SSE2__ - static constexpr size_t SIMD_BYTES = 16; - const __m128i zero16 = _mm_setzero_si128(); - const UInt8 * filt_end_sse = filt_pos + size / SIMD_BYTES * SIMD_BYTES; + /** A slightly more optimized version. + * Based on the assumption that often pieces of consecutive values + * completely pass or do not pass the filter. + * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. + */ + static constexpr size_t SIMD_BYTES = 64; + const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; - while (filt_pos < filt_end_sse) + while (filt_pos < filt_end_aligned) { - UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(filt_pos)), zero16)); - mask = ~mask; - while (mask) + UInt64 mask = Bytes64MaskToBits64Mask(filt_pos); + + if (0xffffffffffffffff == mask) { - size_t index = __builtin_ctz(mask); - res_data.push_back(*(data_pos + index)); - mask = mask & (mask - 1); + res_data.insert(data_pos, data_pos + SIMD_BYTES); } + else + { + while (mask) + { + size_t index = __builtin_ctzll(mask); + res_data.push_back(data_pos[index]); + #ifdef __BMI__ + mask = _blsr_u64(mask); + #else + mask = mask & (mask-1); + #endif + } + } + filt_pos += SIMD_BYTES; data_pos += SIMD_BYTES; } -#endif while (filt_pos < filt_end) { diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 729d82da41d..cc4e7b12a04 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -236,17 +236,15 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result * completely pass or do not pass the filter. * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. */ -#if defined(__AVX512F__) && defined(__AVX512BW__) static constexpr size_t SIMD_BYTES = 64; - const __m512i zero64 = _mm512_setzero_epi32(); - const UInt8 * filt_end_avx512 = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES; + const UInt8 * filt_end_aligned = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES; const size_t chars_per_simd_elements = SIMD_BYTES * n; - while (filt_pos < filt_end_avx512) + while (filt_pos < filt_end_aligned) { - uint64_t mask = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(filt_pos)), zero64, _MM_CMPINT_GT); + uint64_t mask = Bytes64MaskToBits64Mask(filt_pos); - if (0xFFFFFFFFFFFFFFFF == mask) + if (0xffffffffffffffff == mask) { res->chars.insert(data_pos, data_pos + chars_per_simd_elements); } @@ -269,73 +267,6 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result data_pos += chars_per_simd_elements; filt_pos += SIMD_BYTES; } -#elif defined(__AVX__) && defined(__AVX2__) - static constexpr size_t SIMD_BYTES = 32; - const __m256i zero32 = _mm256_setzero_si256(); - const UInt8 * filt_end_avx2 = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES; - const size_t chars_per_simd_elements = SIMD_BYTES * n; - - while (filt_pos < filt_end_avx2) - { - uint32_t mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(_mm256_loadu_si256(reinterpret_cast(filt_pos)), zero32)); - - if (0xFFFFFFFF == mask) - { - res->chars.insert(data_pos, data_pos + chars_per_simd_elements); - } - else - { - size_t res_chars_size = res->chars.size(); - while (mask) - { - size_t index = __builtin_ctz(mask); - res->chars.resize(res_chars_size + n); - memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n); - res_chars_size += n; - #ifdef __BMI__ - mask = _blsr_u32(mask); - #else - mask = mask & (mask-1); - #endif - } - } - data_pos += chars_per_simd_elements; - filt_pos += SIMD_BYTES; - } - -#elif defined(__SSE2__) - - static constexpr size_t SIMD_BYTES = 16; - const __m128i zero16 = _mm_setzero_si128(); - const UInt8 * filt_end_sse = filt_pos + col_size / SIMD_BYTES * SIMD_BYTES; - const size_t chars_per_simd_elements = SIMD_BYTES * n; - - while (filt_pos < filt_end_sse) - { - UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(filt_pos)), zero16)); - mask = ~mask; - - if (0xFFFF == mask) - { - res->chars.insert(data_pos, data_pos + chars_per_simd_elements); - } - else - { - size_t res_chars_size = res->chars.size(); - while (mask) - { - size_t index = __builtin_ctz(mask); - res->chars.resize(res_chars_size + n); - memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n); - res_chars_size += n; - mask = mask & (mask - 1); - } - } - data_pos += chars_per_simd_elements; - filt_pos += SIMD_BYTES; - } - -#endif size_t res_chars_size = res->chars.size(); while (filt_pos < filt_end) diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 4793b6bd9d5..b9079857aa4 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -310,21 +310,20 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s const UInt8 * filt_pos = filt.data(); const UInt8 * filt_end = filt_pos + size; const T * data_pos = data.data(); + /** A slightly more optimized version. * Based on the assumption that often pieces of consecutive values * completely pass or do not pass the filter. * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. */ -#if defined(__AVX512F__) && defined(__AVX512BW__) static constexpr size_t SIMD_BYTES = 64; - const __m512i zero64 = _mm512_setzero_epi32(); - const UInt8 * filt_end_avx512 = filt_pos + size / SIMD_BYTES * SIMD_BYTES; + const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; - while (filt_pos < filt_end_avx512) + while (filt_pos < filt_end_aligned) { - UInt64 mask = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(filt_pos)), zero64, _MM_CMPINT_GT); + UInt64 mask = Bytes64MaskToBits64Mask(filt_pos); - if (0xFFFFFFFFFFFFFFFF == mask) + if (0xffffffffffffffff == mask) { res_data.insert(data_pos, data_pos + SIMD_BYTES); } @@ -346,67 +345,6 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s data_pos += SIMD_BYTES; } -#elif defined(__AVX__) && defined(__AVX2__) - static constexpr size_t SIMD_BYTES = 32; - const __m256i zero32 = _mm256_setzero_si256(); - const UInt8 * filt_end_avx2 = filt_pos + size / SIMD_BYTES * SIMD_BYTES; - - while (filt_pos < filt_end_avx2) - { - UInt32 mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(_mm256_loadu_si256(reinterpret_cast(filt_pos)), zero32)); - - if (0xFFFFFFFF == mask) - { - res_data.insert(data_pos, data_pos + SIMD_BYTES); - } - else - { - while (mask) - { - size_t index = __builtin_ctz(mask); - res_data.push_back(data_pos[index]); - #ifdef __BMI__ - mask = _blsr_u32(mask); - #else - mask = mask & (mask-1); - #endif - } - } - - filt_pos += SIMD_BYTES; - data_pos += SIMD_BYTES; - } - -#elif defined(__SSE2__) - static constexpr size_t SIMD_BYTES = 16; - const __m128i zero16 = _mm_setzero_si128(); - const UInt8 * filt_end_sse = filt_pos + size / SIMD_BYTES * SIMD_BYTES; - - while (filt_pos < filt_end_sse) - { - UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(filt_pos)), zero16)); - mask = ~mask; - - if (0xFFFF == mask) - { - res_data.insert(data_pos, data_pos + SIMD_BYTES); - } - else - { - while (mask) - { - size_t index = __builtin_ctz(mask); - res_data.push_back(data_pos[index]); - mask = mask & (mask - 1); - } - } - - filt_pos += SIMD_BYTES; - data_pos += SIMD_BYTES; - } - -#endif - while (filt_pos < filt_end) { if (*filt_pos) diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index ea5ca4e45b4..5b9d6344aee 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -12,24 +12,39 @@ namespace DB { -#if defined(__SSE2__) && defined(__POPCNT__) -/// Transform 64-byte mask to 64-bit mask. -static UInt64 toBits64(const Int8 * bytes64) +/// Transform 64-byte mask to 64-bit mask +UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64) { +#if defined(__AVX512F__) && defined(__AVX512BW__) + static const __m512i zero64 = _mm512_setzero_epi32(); + UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_GT); +#elif defined(__AVX__) && defined(__AVX2__) + static const __m256i zero32 = _mm256_setzero_si256(); + UInt64 res = + (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + _mm256_loadu_si256(reinterpret_cast(bytes64)), zero32))) & 0xffffffff) + | (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + _mm256_loadu_si256(reinterpret_cast(bytes64+32)), zero32))) << 32); +#elif defined(__SSE2__) && defined(__POPCNT__) static const __m128i zero16 = _mm_setzero_si128(); UInt64 res = - static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) - | (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) - | (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) - | (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48); - - return ~res; -} + static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48); +#else + UInt64 res = 0; + const UInt8 * pos = bytes64; + const UInt8 * end = pos + 64; + for (; pos < end; ++pos) + res |= (*pos != 0)<<(pos-bytes64)); #endif + return res; +} size_t countBytesInFilter(const UInt8 * filt, size_t sz) { @@ -40,14 +55,14 @@ size_t countBytesInFilter(const UInt8 * filt, size_t sz) * It would be better to use != 0, then this does not allow SSE2. */ - const Int8 * pos = reinterpret_cast(filt); - const Int8 * end = pos + sz; + const UInt8 * pos = filt; + const UInt8 * end = pos + sz; #if defined(__SSE2__) && defined(__POPCNT__) - const Int8 * end64 = pos + sz / 64 * 64; + const UInt8 * end64 = pos + sz / 64 * 64; for (; pos < end64; pos += 64) - count += __builtin_popcountll(toBits64(pos)); + count += __builtin_popcountll(Bytes64MaskToBits64Mask(pos)); /// TODO Add duff device for tail? #endif @@ -72,15 +87,15 @@ size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * nu * It would be better to use != 0, then this does not allow SSE2. */ - const Int8 * pos = reinterpret_cast(filt.data()); - const Int8 * pos2 = reinterpret_cast(null_map); - const Int8 * end = pos + filt.size(); + const UInt8 * pos = filt.data(); + const UInt8 * pos2 = null_map; + const UInt8 * end = pos + filt.size(); #if defined(__SSE2__) && defined(__POPCNT__) - const Int8 * end64 = pos + filt.size() / 64 * 64; + const UInt8 * end64 = pos + filt.size() / 64 * 64; for (; pos < end64; pos += 64, pos2 += 64) - count += __builtin_popcountll(toBits64(pos) & ~toBits64(pos2)); + count += __builtin_popcountll(Bytes64MaskToBits64Mask(pos) & ~Bytes64MaskToBits64Mask(pos2)); /// TODO Add duff device for tail? #endif diff --git a/src/Columns/ColumnsCommon.h b/src/Columns/ColumnsCommon.h index 25f87e90bba..baf26deb00c 100644 --- a/src/Columns/ColumnsCommon.h +++ b/src/Columns/ColumnsCommon.h @@ -15,6 +15,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +/// Transform 64-byte mask to 64-bit mask +UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64); /// Counts how many bytes of `filt` are greater than zero. size_t countBytesInFilter(const UInt8 * filt, size_t sz); size_t countBytesInFilter(const IColumn::Filter & filt); From 3810baf44cd63ded64f886d38aea10d6a05450e0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 26 Oct 2021 00:00:57 +0300 Subject: [PATCH 082/173] CompiledExpressionCache limit elements size --- programs/server/Server.cpp | 7 ++++++- programs/server/config.xml | 5 ++++- programs/server/config.yaml.example | 5 ++++- src/Common/LRUCache.h | 14 +++++++------- src/Interpreters/JIT/CompiledExpressionCache.cpp | 4 ++-- src/Interpreters/JIT/CompiledExpressionCache.h | 2 +- 6 files changed, 24 insertions(+), 13 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index d850ca45e26..2be9e57e437 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -957,9 +957,14 @@ if (ThreadFuzzer::instance().isEffective()) global_context->setMMappedFileCache(mmap_cache_size); #if USE_EMBEDDED_COMPILER + /// 128 MB constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 128; size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", compiled_expression_cache_size_default); - CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_size); + + constexpr size_t compiled_expression_cache_elements_size_default = 10000; + size_t compiled_expression_cache_elements_size = config().getUInt64("compiled_expression_cache_elements_size", compiled_expression_cache_elements_size_default); + + CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_size, compiled_expression_cache_elements_size); #endif /// Set path for format schema files diff --git a/programs/server/config.xml b/programs/server/config.xml index e38a6daeaed..37f36aa5215 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -351,9 +351,12 @@ --> 1000 - + 134217728 + + 10000 + /var/lib/clickhouse/ diff --git a/programs/server/config.yaml.example b/programs/server/config.yaml.example index b2d016e06e5..c312e6a2208 100644 --- a/programs/server/config.yaml.example +++ b/programs/server/config.yaml.example @@ -279,9 +279,12 @@ mark_cache_size: 5368709120 # also it can be dropped manually by the SYSTEM DROP MMAP CACHE query. mmap_cache_size: 1000 -# Cache size for compiled expressions. +# Cache size in bytes for compiled expressions. compiled_expression_cache_size: 134217728 +# Cache size in elements for compiled expressions. +compiled_expression_cache_elements_size: 10000 + # Path to data directory, with trailing slash. path: /var/lib/clickhouse/ diff --git a/src/Common/LRUCache.h b/src/Common/LRUCache.h index 93abfce3c4e..3df27cbdce9 100644 --- a/src/Common/LRUCache.h +++ b/src/Common/LRUCache.h @@ -36,12 +36,10 @@ public: using Mapped = TMapped; using MappedPtr = std::shared_ptr; -private: - using Clock = std::chrono::steady_clock; - -public: - LRUCache(size_t max_size_) - : max_size(std::max(static_cast(1), max_size_)) {} + LRUCache(size_t max_size_, size_t max_elements_size_ = 0) + : max_size(std::max(static_cast(1), max_size_)) + , max_elements_size(max_elements_size_) + {} MappedPtr get(const Key & key) { @@ -252,6 +250,7 @@ private: /// Total weight of values. size_t current_size = 0; const size_t max_size; + const size_t max_elements_size; std::atomic hits {0}; std::atomic misses {0}; @@ -311,7 +310,8 @@ private: { size_t current_weight_lost = 0; size_t queue_size = cells.size(); - while ((current_size > max_size) && (queue_size > 1)) + + while ((current_size > max_size || (max_elements_size != 0 && queue_size > max_elements_size)) && (queue_size > 1)) { const Key & key = queue.front(); diff --git a/src/Interpreters/JIT/CompiledExpressionCache.cpp b/src/Interpreters/JIT/CompiledExpressionCache.cpp index 98f4eec982d..674e02236f5 100644 --- a/src/Interpreters/JIT/CompiledExpressionCache.cpp +++ b/src/Interpreters/JIT/CompiledExpressionCache.cpp @@ -16,12 +16,12 @@ CompiledExpressionCacheFactory & CompiledExpressionCacheFactory::instance() return factory; } -void CompiledExpressionCacheFactory::init(size_t cache_size) +void CompiledExpressionCacheFactory::init(size_t cache_size_in_bytes, size_t cache_size_in_elements) { if (cache) throw Exception(ErrorCodes::LOGICAL_ERROR, "CompiledExpressionCache was already initialized"); - cache = std::make_unique(cache_size); + cache = std::make_unique(cache_size_in_bytes, cache_size_in_elements); } CompiledExpressionCache * CompiledExpressionCacheFactory::tryGetCache() diff --git a/src/Interpreters/JIT/CompiledExpressionCache.h b/src/Interpreters/JIT/CompiledExpressionCache.h index 5182a77d77a..7d20627d5d2 100644 --- a/src/Interpreters/JIT/CompiledExpressionCache.h +++ b/src/Interpreters/JIT/CompiledExpressionCache.h @@ -52,7 +52,7 @@ private: public: static CompiledExpressionCacheFactory & instance(); - void init(size_t cache_size); + void init(size_t cache_size_in_bytes, size_t cache_size_in_elements); CompiledExpressionCache * tryGetCache(); }; From 16cd70aee908b2001649650e4def185d217393fe Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 22 Oct 2021 00:12:30 +0300 Subject: [PATCH 083/173] Fix PREWHERE with WHERE in case of always true PREWHERE --- src/Storages/MergeTree/MergeTreeRangeReader.cpp | 13 +++++++++---- .../02021_prewhere_always_true_where.reference | 1 + .../02021_prewhere_always_true_where.sql | 5 +++++ 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02021_prewhere_always_true_where.reference create mode 100644 tests/queries/0_stateless/02021_prewhere_always_true_where.sql diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 2f6bc10e472..c5454f33c79 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1044,10 +1044,15 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r /// Filter in WHERE instead else { - auto type = getSampleBlock().getByName(prewhere_info->prewhere_column_name).type; - ColumnWithTypeAndName col(result.getFilterHolder()->convertToFullColumnIfConst(), std::make_shared(), ""); - result.columns[prewhere_column_pos] = castColumn(col, type); - result.clearFilter(); // Acting as a flag to not filter in PREWHERE + if (prewhere_info->remove_prewhere_column) + result.columns.erase(result.columns.begin() + prewhere_column_pos); + else + { + auto type = getSampleBlock().getByName(prewhere_info->prewhere_column_name).type; + ColumnWithTypeAndName col(result.getFilterHolder()->convertToFullColumnIfConst(), std::make_shared(), ""); + result.columns[prewhere_column_pos] = castColumn(col, type); + result.clearFilter(); // Acting as a flag to not filter in PREWHERE + } } } diff --git a/tests/queries/0_stateless/02021_prewhere_always_true_where.reference b/tests/queries/0_stateless/02021_prewhere_always_true_where.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02021_prewhere_always_true_where.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02021_prewhere_always_true_where.sql b/tests/queries/0_stateless/02021_prewhere_always_true_where.sql new file mode 100644 index 00000000000..95dcb6a15c2 --- /dev/null +++ b/tests/queries/0_stateless/02021_prewhere_always_true_where.sql @@ -0,0 +1,5 @@ +drop table if exists data_02021; +create table data_02021 (key Int) engine=MergeTree() order by key; +insert into data_02021 values (1); +select count() from data_02021 prewhere 1 or ignore(key) where ignore(key)=0; +drop table data_02021; From 911b9c3973fb75c130392a5c1e89e83e42f4236e Mon Sep 17 00:00:00 2001 From: jasperzhu Date: Tue, 26 Oct 2021 06:02:57 +0530 Subject: [PATCH 084/173] Improve function defect --- src/Columns/ColumnsCommon.cpp | 119 ++++++---------------------------- 1 file changed, 20 insertions(+), 99 deletions(-) diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index 5b9d6344aee..2be9b18de4e 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -17,33 +17,33 @@ UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64) { #if defined(__AVX512F__) && defined(__AVX512BW__) static const __m512i zero64 = _mm512_setzero_epi32(); - UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_GT); + UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_EQ); #elif defined(__AVX__) && defined(__AVX2__) static const __m256i zero32 = _mm256_setzero_si256(); UInt64 res = - (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8( _mm256_loadu_si256(reinterpret_cast(bytes64)), zero32))) & 0xffffffff) - | (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + | (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8( _mm256_loadu_si256(reinterpret_cast(bytes64+32)), zero32))) << 32); #elif defined(__SSE2__) && defined(__POPCNT__) static const __m128i zero16 = _mm_setzero_si128(); UInt64 res = - static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) - | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) - | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) - | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48); + (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) & 0xffff) + | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) & 0xffff0000) + | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) & 0xffff00000000) + | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48) & 0xffff000000000000); #else UInt64 res = 0; const UInt8 * pos = bytes64; const UInt8 * end = pos + 64; for (; pos < end; ++pos) - res |= (*pos != 0)<<(pos-bytes64)); + res |= (*pos == 0)<<(pos-bytes64)); #endif - return res; + return ~res; } size_t countBytesInFilter(const UInt8 * filt, size_t sz) @@ -244,16 +244,19 @@ namespace memcpy(&res_elems[elems_size_old], &src_elems[arr_offset], arr_size * sizeof(T)); }; - #if defined(__AVX512F__) && defined(__AVX512BW__) - const __m512i zero_vec = _mm512_setzero_epi32(); + /** A slightly more optimized version. + * Based on the assumption that often pieces of consecutive values + * completely pass or do not pass the filter. + * Therefore, we will optimistically check the parts of `SIMD_BYTES` values. + */ static constexpr size_t SIMD_BYTES = 64; const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; while (filt_pos < filt_end_aligned) { - uint64_t mask = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(filt_pos)), zero_vec, _MM_CMPINT_GT); + uint64_t mask = Bytes64MaskToBits64Mask(filt_pos); - if (mask == 0xffffffffffffffff) + if (0xffffffffffffffff == mask) { /// SIMD_BYTES consecutive rows pass the filter const auto first = offsets_pos == offsets_begin; @@ -285,88 +288,6 @@ namespace filt_pos += SIMD_BYTES; offsets_pos += SIMD_BYTES; } - #elif defined(__AVX__) && defined(__AVX2__) - const __m256i zero_vec = _mm256_setzero_si256(); - static constexpr size_t SIMD_BYTES = 32; - const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; - - while (filt_pos < filt_end_aligned) - { - uint32_t mask = _mm256_movemask_epi8(_mm256_cmpgt_epi8(_mm256_loadu_si256(reinterpret_cast(filt_pos)), zero_vec)); - - if (mask == 0xffffffff) - { - /// SIMD_BYTES consecutive rows pass the filter - const auto first = offsets_pos == offsets_begin; - - const auto chunk_offset = first ? 0 : offsets_pos[-1]; - const auto chunk_size = offsets_pos[SIMD_BYTES - 1] - chunk_offset; - - result_offsets_builder.template insertChunk(offsets_pos, first, chunk_offset, chunk_size); - - /// copy elements for SIMD_BYTES arrays at once - const auto elems_size_old = res_elems.size(); - res_elems.resize(elems_size_old + chunk_size); - memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T)); - } - else - { - while (mask) - { - size_t index = __builtin_ctz(mask); - copy_array(offsets_pos + index); - #ifdef __BMI__ - mask = _blsr_u32(mask); - #else - mask = mask & (mask-1); - #endif - } - } - - filt_pos += SIMD_BYTES; - offsets_pos += SIMD_BYTES; - } - #elif defined(__SSE2__) - const __m128i zero_vec = _mm_setzero_si128(); - static constexpr size_t SIMD_BYTES = 16; - const auto * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES; - - while (filt_pos < filt_end_aligned) - { - UInt16 mask = _mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(filt_pos)), - zero_vec)); - mask = ~mask; - - if (mask == 0xffff) - { - /// SIMD_BYTES consecutive rows pass the filter - const auto first = offsets_pos == offsets_begin; - - const auto chunk_offset = first ? 0 : offsets_pos[-1]; - const auto chunk_size = offsets_pos[SIMD_BYTES - 1] - chunk_offset; - - result_offsets_builder.template insertChunk(offsets_pos, first, chunk_offset, chunk_size); - - /// copy elements for SIMD_BYTES arrays at once - const auto elems_size_old = res_elems.size(); - res_elems.resize(elems_size_old + chunk_size); - memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T)); - } - else - { - while (mask) - { - size_t index = __builtin_ctz(mask); - copy_array(offsets_pos + index); - mask = mask & (mask - 1); - } - } - - filt_pos += SIMD_BYTES; - offsets_pos += SIMD_BYTES; - } - #endif while (filt_pos < filt_end) { From 6e3921f77e3e52e6b9b15e6d861a2f1e7924a13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 09:12:13 +0800 Subject: [PATCH 085/173] delete symbolic links --- .../external-authenticators/kerberos.md | 105 ------------------ 1 file changed, 105 deletions(-) delete mode 120000 docs/zh/operations/external-authenticators/kerberos.md diff --git a/docs/zh/operations/external-authenticators/kerberos.md b/docs/zh/operations/external-authenticators/kerberos.md deleted file mode 120000 index 1121a86a793..00000000000 --- a/docs/zh/operations/external-authenticators/kerberos.md +++ /dev/null @@ -1,105 +0,0 @@ -# Kerberos认证 {#external-authenticators-kerberos} -现有正确配置的 ClickHouse 用户可以通过 Kerberos 身份验证协议进行身份验证. - -目前, Kerberos 只能用作现有用户的外部身份验证器，这些用户在 `users.xml` 或本地访问控制路径中定义. -这些用户只能使用 HTTP 请求, 并且必须能够使用 GSS-SPNEGO 机制进行身份验证. - -对于这种方法, 必须在系统中配置 Kerberos, 且必须在 ClickHouse 配置中启用. - -## 开启Kerberos {#enabling-kerberos-in-clickHouse} -要启用 Kerberos, 应该在 `config.xml` 中包含 `kerberos` 部分. 此部分可能包含其他参数. - -#### 参数: {#parameters} -- `principal` - 将在接受安全上下文时获取和使用的规范服务主体名称. -- 此参数是可选的, 如果省略, 将使用默认主体. - -- `realm` - 一个领域, 用于将身份验证限制为仅那些发起者领域与其匹配的请求. - -- 此参数是可选的，如果省略，则不会应用其他领域的过滤. - -示例 (进入 `config.xml`): -```xml - - - - -``` - -主体规范: -```xml - - - - HTTP/clickhouse.example.com@EXAMPLE.COM - - -``` - -按领域过滤: -```xml - - - - EXAMPLE.COM - - -``` - -!!! warning "注意" - -您只能定义一个 `kerberos` 部分. 多个 `kerberos` 部分的存在将强制 ClickHouse 禁用 Kerberos 身份验证. - -!!! warning "注意" - -`主体`和`领域`部分不能同时指定. `主体`和`领域`的出现将迫使ClickHouse禁用Kerberos身份验证. - -## Kerberos作为现有用户的外部身份验证器 {#kerberos-as-an-external-authenticator-for-existing-users} -Kerberos可以用作验证本地定义用户(在`users.xml`或本地访问控制路径中定义的用户)身份的方法。目前，**只有**通过HTTP接口的请求才能被认证(通过GSS-SPNEGO机制). - -Kerberos主体名称格式通常遵循以下模式: -- *primary/instance@REALM* - -*/instance* 部分可能出现零次或多次. **发起者的规范主体名称的主要部分应与被认证用户名匹配, 以便身份验证成功**. - -### `users.xml`中启用Kerberos {#enabling-kerberos-in-users-xml} -为了启用用户的 Kerberos 身份验证, 请在用户定义中指定 `kerberos` 部分而不是`密码`或类似部分. - -参数: -- `realm` - 用于将身份验证限制为仅那些发起者的领域与其匹配的请求的领域. -- 此参数是可选的, 如果省略, 则不会应用其他按领域的过滤. - -示例 (进入 `users.xml`): -``` - - - - - - - - EXAMPLE.COM - - - - -``` - -!!! warning "警告" - -注意, Kerberos身份验证不能与任何其他身份验证机制一起使用. 任何其他部分(如`密码`和`kerberos`)的出现都会迫使ClickHouse关闭. - -!!! info "提醒" - -请注意, 现在, 一旦用户 `my_user` 使用 `kerberos`, 必须在主 `config.xml` 文件中启用 Kerberos，如前所述. - -### 使用 SQL 启用 Kerberos {#enabling-kerberos-using-sql} -在 ClickHouse 中启用 [SQL 驱动的访问控制和帐户管理](./access-rights.md#access-control)后, 也可以使用 SQL 语句创建由 Kerberos 识别的用户. - -```sql -CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' -``` - -...或者, 不按领域过滤: -```sql -CREATE USER my_user IDENTIFIED WITH kerberos -``` From 37d7eb0cb82fa47b453795fcffdef934d9c4791c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 09:13:26 +0800 Subject: [PATCH 086/173] replace symbolic links --- .../external-authenticators/kerberos | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 docs/zh/operations/external-authenticators/kerberos diff --git a/docs/zh/operations/external-authenticators/kerberos b/docs/zh/operations/external-authenticators/kerberos new file mode 100644 index 00000000000..1121a86a793 --- /dev/null +++ b/docs/zh/operations/external-authenticators/kerberos @@ -0,0 +1,105 @@ +# Kerberos认证 {#external-authenticators-kerberos} +现有正确配置的 ClickHouse 用户可以通过 Kerberos 身份验证协议进行身份验证. + +目前, Kerberos 只能用作现有用户的外部身份验证器，这些用户在 `users.xml` 或本地访问控制路径中定义. +这些用户只能使用 HTTP 请求, 并且必须能够使用 GSS-SPNEGO 机制进行身份验证. + +对于这种方法, 必须在系统中配置 Kerberos, 且必须在 ClickHouse 配置中启用. + +## 开启Kerberos {#enabling-kerberos-in-clickHouse} +要启用 Kerberos, 应该在 `config.xml` 中包含 `kerberos` 部分. 此部分可能包含其他参数. + +#### 参数: {#parameters} +- `principal` - 将在接受安全上下文时获取和使用的规范服务主体名称. +- 此参数是可选的, 如果省略, 将使用默认主体. + +- `realm` - 一个领域, 用于将身份验证限制为仅那些发起者领域与其匹配的请求. + +- 此参数是可选的，如果省略，则不会应用其他领域的过滤. + +示例 (进入 `config.xml`): +```xml + + + + +``` + +主体规范: +```xml + + + + HTTP/clickhouse.example.com@EXAMPLE.COM + + +``` + +按领域过滤: +```xml + + + + EXAMPLE.COM + + +``` + +!!! warning "注意" + +您只能定义一个 `kerberos` 部分. 多个 `kerberos` 部分的存在将强制 ClickHouse 禁用 Kerberos 身份验证. + +!!! warning "注意" + +`主体`和`领域`部分不能同时指定. `主体`和`领域`的出现将迫使ClickHouse禁用Kerberos身份验证. + +## Kerberos作为现有用户的外部身份验证器 {#kerberos-as-an-external-authenticator-for-existing-users} +Kerberos可以用作验证本地定义用户(在`users.xml`或本地访问控制路径中定义的用户)身份的方法。目前，**只有**通过HTTP接口的请求才能被认证(通过GSS-SPNEGO机制). + +Kerberos主体名称格式通常遵循以下模式: +- *primary/instance@REALM* + +*/instance* 部分可能出现零次或多次. **发起者的规范主体名称的主要部分应与被认证用户名匹配, 以便身份验证成功**. + +### `users.xml`中启用Kerberos {#enabling-kerberos-in-users-xml} +为了启用用户的 Kerberos 身份验证, 请在用户定义中指定 `kerberos` 部分而不是`密码`或类似部分. + +参数: +- `realm` - 用于将身份验证限制为仅那些发起者的领域与其匹配的请求的领域. +- 此参数是可选的, 如果省略, 则不会应用其他按领域的过滤. + +示例 (进入 `users.xml`): +``` + + + + + + + + EXAMPLE.COM + + + + +``` + +!!! warning "警告" + +注意, Kerberos身份验证不能与任何其他身份验证机制一起使用. 任何其他部分(如`密码`和`kerberos`)的出现都会迫使ClickHouse关闭. + +!!! info "提醒" + +请注意, 现在, 一旦用户 `my_user` 使用 `kerberos`, 必须在主 `config.xml` 文件中启用 Kerberos，如前所述. + +### 使用 SQL 启用 Kerberos {#enabling-kerberos-using-sql} +在 ClickHouse 中启用 [SQL 驱动的访问控制和帐户管理](./access-rights.md#access-control)后, 也可以使用 SQL 语句创建由 Kerberos 识别的用户. + +```sql +CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' +``` + +...或者, 不按领域过滤: +```sql +CREATE USER my_user IDENTIFIED WITH kerberos +``` From 8ec04169b68039d51243aa6e117dc4bc5b4d633e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 09:14:17 +0800 Subject: [PATCH 087/173] replace symbolic links --- .../operations/external-authenticators/{kerberos => kerberos.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/zh/operations/external-authenticators/{kerberos => kerberos.md} (100%) diff --git a/docs/zh/operations/external-authenticators/kerberos b/docs/zh/operations/external-authenticators/kerberos.md similarity index 100% rename from docs/zh/operations/external-authenticators/kerberos rename to docs/zh/operations/external-authenticators/kerberos.md From 5dc6c7a157f308bbee6f5f6fd056bf4ceb28e24a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 09:32:06 +0800 Subject: [PATCH 088/173] adjust title level --- docs/zh/development/continuous-integration.md | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 7e4c1bf36a9..41e22074915 120000 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -17,50 +17,50 @@ git push 如果您不确定要做什么，可以向维护人员寻求帮助. -# 与Master合并 {#merge-with-master} +## 与Master合并 {#merge-with-master} 验证PR是否可以合并到master. 如果没有, 它将失败并显示消息'Cannot fetch mergecommit'的.请按[GitHub文档](https://docs.github.com/en/github/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-on-github)中描述的冲突解决, 或使用git将主分支合并到您的pull请求分支来修复这个检查. -# 文档检查 {#docs-check} +## 文档检查 {#docs-check} 尝试构建ClickHouse文档网站. 如果您更改了文档中的某些内容, 它可能会失败. 最可能的原因是文档中的某些交叉链接是错误的. 转到检查报告并查找`ERROR`和`WARNING`消息. -# 报告详情 {#report-details} +### 报告详情 {#report-details} - [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) - `docs_output.txt`包含构建日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) -# 描述信息检查 {#description-check} +## 描述信息检查 {#description-check} 检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](../../../.github/PULL_REQUEST_TEMPLATE.md)模板. 您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](../whats-new/changelog/)编写一条用户可读的消息用来描述更改. -# 推送到DockerHub {#push-to-dockerhub} +## 推送到DockerHub {#push-to-dockerhub} 生成用于构建和测试的docker映像, 然后将它们推送到DockerHub. -# 标记检查 {#marker-check} +## 标记检查 {#marker-check} 该检查意味着CI系统已经开始处理PR.当它处于'待处理'状态时，意味着尚未开始所有检查. 启动所有检查后，状态更改为'成功'. # 格式检查 {#style-check} 使用`utils/check-style/check-style`二进制文件执行一些简单的基于正则表达式的代码样式检查(注意, 它可以在本地运行). 如果失败, 按照[代码样式指南](./style.md)修复样式错误. -# 报告详情 {#report-details} +### 报告详情 {#report-details} - [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) - `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt) -# PVS 检查 {#pvs-check} +### PVS 检查 {#pvs-check} 使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告以查看确切的错误.如果可以则修复它们, 如果不行, 可以向ClickHouse的维护人员寻求帮忙. -# 报告详情 {#report-details} +### 报告详情 {#report-details} - [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) - `test_run.txt.out.log`包含构建和分析日志文件.它只包含解析或未找到的错误. - `HTML report`包含分析结果.有关说明请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) -# 快速测试 {#fast-test} +## 快速测试 {#fast-test} 通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 查看报告以了解哪些测试失败, 然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败. -# 报告详情 {#report-details} +### 报告详情 {#report-details} [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/fast_test.html) -# 状态页文件 {#status-page-files} +#### 状态页文件 {#status-page-files} - `runlog.out.log` 是包含所有其他日志的通用日志. - `test_log.txt` - `submodule_log.txt` 包含关于克隆和检查所需子模块的消息. @@ -73,15 +73,15 @@ git push - `build_log.txt` - `cmake_log.txt` 包含关于C/C++和Linux标志检查的消息. -# 状态页列信息 {#status-page-columns} +#### 状态页列信息 {#status-page-columns} - 测试名称 -- 包含测试的名称(不带路径, 例如, 所有类型的测试将被剥离到该名称). - 测试状态 -- 跳过、成功或失败之一. - 测试时间, 秒. -- 这个测试是空的. -# 建构检查 {#build-check} +## 建构检查 {#build-check} 在各种配置中构建ClickHouse, 以便在后续步骤中使用. 您必须修复失败的构建.构建日志通常有足够的信息来修复错误, 但是您可能必须在本地重现故障. `cmake`选项可以在构建日志中通过grep `cmake`操作找到.使用这些选项并遵循[一般的构建过程](./build.md). -# 报告详情 {#report-details} +### 报告详情 {#report-details} [状态页示例](https://clickhouse-builds.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/clickhouse_build_check/report.html) - **Compiler**: `gcc-9` 或 `clang-10` (或其他架构的`clang-10-xx`, 比如`clang-10-freebsd`). - **Build type**: `Debug` or `RelWithDebInfo` (cmake). @@ -105,22 +105,22 @@ git push - `shared_build.tgz`: 使用共享库构建. - `performance.tgz`: 用于性能测试的特殊包. -# 特殊构建检查 {#special-buildcheck} +## 特殊构建检查 {#special-buildcheck} 使用clang-tidy执行静态分析和代码样式检查. 该报告类似于构建检查. 修复在构建日志中发现的错误. -# 功能无状态测试 {#functional-stateless-tests} +## 功能无状态测试 {#functional-stateless-tests} 为构建在不同配置中的ClickHouse二进制文件运行[无状态功能测试](./tests.md#functional-tests)——发布、调试、使用杀毒软件等.通过报告查看哪些测试失败，然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败.注意, 您必须使用正确的构建配置来重现——在AddressSanitizer下测试可能失败,但在Debug中可以通过.从[CI构建检查页面](./build.md#you-dont-have-to-build-clickhouse)下载二进制文件, 或者在本地构建它. -# 功能有状态测试 {#functional-stateful-tests} +## 功能有状态测试 {#functional-stateful-tests} 运行[有状态功能测试](./tests.md#functional-tests).以无状态功能测试相同的方式对待它们.不同之处在于它们需要从[Yandex.Metrica数据集](https://clickhouse.com/docs/en/getting-started/example-datasets/metrica/)的`hits`和`visits`表来运行. -# 集成测试 {#integration-tests} +## 集成测试 {#integration-tests} 运行[集成测试](./tests.md#integration-tests). -# Testflows 检查{#testflows-check} +## Testflows 检查{#testflows-check} 使用Testflows测试系统去运行一些测试, 在[此处](../../../tests/testflows/README.md#running-tests-locally)查看如何在本地运行它们. -# 压力测试 {#stress-test} +## 压力测试 {#stress-test} 从多个客户端并发运行无状态功能测试, 用以检测与并发相关的错误.如果失败: ``` * Fix all other test failures first; @@ -128,7 +128,7 @@ git push of error. ``` -# 冒烟测试 {#split-build-smoke-test} +## 冒烟测试 {#split-build-smoke-test} 检查[拆分构建](./build.md#split-build)配置中的服务器构建是否可以启动并运行简单查询.如果失败: ``` * Fix other test errors first; @@ -136,16 +136,16 @@ git push locally and check whether it can start and run `select 1`. ``` -# 兼容性检查 {#compatibility-check} +## 兼容性检查 {#compatibility-check} 检查`clickhouse`二进制文件是否可以在带有旧libc版本的发行版上运行.如果失败, 请向维护人员寻求帮助. -# AST模糊器 {#ast-fuzzer} +## AST模糊器 {#ast-fuzzer} 运行随机生成的查询来捕获程序错误.如果失败, 请向维护人员寻求帮助. -# 性能测试 {#performance-tests} +## 性能测试 {#performance-tests} 测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](../../../docker/test/performance-comparison/README.md#how-to-read-the-report)有详细描述. -# 质量保证 {#qa} +## 质量保证 {#qa} 什么是状态页面上的任务(专用网络)项目? 它是 Yandex 内部工作系统的链接. Yandex 员工可以看到检查的开始时间及其更详细的状态. From aa4312760128c70cb606d364ac8ce545c5a06c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 09:33:16 +0800 Subject: [PATCH 089/173] delete symbolic links --- docs/zh/development/continuous-integration.md | 155 ------------------ 1 file changed, 155 deletions(-) delete mode 120000 docs/zh/development/continuous-integration.md diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md deleted file mode 120000 index 41e22074915..00000000000 --- a/docs/zh/development/continuous-integration.md +++ /dev/null @@ -1,155 +0,0 @@ -# 持续集成检查 {#continuous-integration-checks} -当你提交一个pull请求时, ClickHouse[持续集成(CI)系统](../tests.md#test-automation)会对您的代码运行一些自动检查. - -这在存储库维护者(来自ClickHouse团队的人)筛选了您的代码并将可测试标签添加到您的pull请求之后发生. - -检查的结果被列在[GitHub检查文档](https://docs.github.com/en/github/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks)中所述的GitHub pull请求页面. - -如果检查失败，您可能被要求去修复它. 该界面介绍了您可能遇到的检查，以及如何修复它们. - -如果检查失败看起来与您的更改无关, 那么它可能是一些暂时的故障或基础设施问题. 向pull请求推一个空的commit以重新启动CI检查: - -``` -git reset -git commit --allow-empty -git push -``` - -如果您不确定要做什么，可以向维护人员寻求帮助. - -## 与Master合并 {#merge-with-master} -验证PR是否可以合并到master. 如果没有, 它将失败并显示消息'Cannot fetch mergecommit'的.请按[GitHub文档](https://docs.github.com/en/github/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-on-github)中描述的冲突解决, 或使用git将主分支合并到您的pull请求分支来修复这个检查. - -## 文档检查 {#docs-check} -尝试构建ClickHouse文档网站. 如果您更改了文档中的某些内容, 它可能会失败. 最可能的原因是文档中的某些交叉链接是错误的. 转到检查报告并查找`ERROR`和`WARNING`消息. - -### 报告详情 {#report-details} -- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) -- `docs_output.txt`包含构建日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) - -## 描述信息检查 {#description-check} -检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](../../../.github/PULL_REQUEST_TEMPLATE.md)模板. - -您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](../whats-new/changelog/)编写一条用户可读的消息用来描述更改. - -## 推送到DockerHub {#push-to-dockerhub} -生成用于构建和测试的docker映像, 然后将它们推送到DockerHub. - -## 标记检查 {#marker-check} -该检查意味着CI系统已经开始处理PR.当它处于'待处理'状态时，意味着尚未开始所有检查. 启动所有检查后，状态更改为'成功'. - -# 格式检查 {#style-check} -使用`utils/check-style/check-style`二进制文件执行一些简单的基于正则表达式的代码样式检查(注意, 它可以在本地运行). -如果失败, 按照[代码样式指南](./style.md)修复样式错误. - -### 报告详情 {#report-details} -- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) -- `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt) - -### PVS 检查 {#pvs-check} -使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告以查看确切的错误.如果可以则修复它们, 如果不行, 可以向ClickHouse的维护人员寻求帮忙. - -### 报告详情 {#report-details} -- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) -- `test_run.txt.out.log`包含构建和分析日志文件.它只包含解析或未找到的错误. -- `HTML report`包含分析结果.有关说明请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) - -## 快速测试 {#fast-test} -通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 查看报告以了解哪些测试失败, 然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败. - -### 报告详情 {#report-details} -[状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/fast_test.html) - -#### 状态页文件 {#status-page-files} -- `runlog.out.log` 是包含所有其他日志的通用日志. -- `test_log.txt` -- `submodule_log.txt` 包含关于克隆和检查所需子模块的消息. -- `stderr.log` -- `stdout.log` -- `clickhouse-server.log` -- `clone_log.txt` -- `install_log.txt` -- `clickhouse-server.err.log` -- `build_log.txt` -- `cmake_log.txt` 包含关于C/C++和Linux标志检查的消息. - -#### 状态页列信息 {#status-page-columns} -- 测试名称 -- 包含测试的名称(不带路径, 例如, 所有类型的测试将被剥离到该名称). -- 测试状态 -- 跳过、成功或失败之一. -- 测试时间, 秒. -- 这个测试是空的. - -## 建构检查 {#build-check} -在各种配置中构建ClickHouse, 以便在后续步骤中使用. 您必须修复失败的构建.构建日志通常有足够的信息来修复错误, 但是您可能必须在本地重现故障. `cmake`选项可以在构建日志中通过grep `cmake`操作找到.使用这些选项并遵循[一般的构建过程](./build.md). - -### 报告详情 {#report-details} -[状态页示例](https://clickhouse-builds.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/clickhouse_build_check/report.html) -- **Compiler**: `gcc-9` 或 `clang-10` (或其他架构的`clang-10-xx`, 比如`clang-10-freebsd`). -- **Build type**: `Debug` or `RelWithDebInfo` (cmake). -- **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan). -- **Bundled**: `bundled` 构建使用来自 `contrib` 库, 而 `unbundled` 构建使用系统库. -- **Splitted**: `splitted` is a [split build](https://clickhouse.com/docs/en/development/build/#split-build) -- **Status**: `成功` 或 `失败` -- **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用. -- **Build time**. -- **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`). - - `clickhouse-client_XXX_all.deb` - -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` - - `clickhouse-common-staticXXX_amd64.deb` - - `clickhouse-server_XXX_all.deb` - - `clickhouse-test_XXX_all.deb` - - `clickhouse_XXX_amd64.buildinfo` - - `clickhouse_XXX_amd64.changes` - - `clickhouse`: Main built binary. - - `clickhouse-odbc-bridge` - - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件. - - `shared_build.tgz`: 使用共享库构建. - - `performance.tgz`: 用于性能测试的特殊包. - -## 特殊构建检查 {#special-buildcheck} -使用clang-tidy执行静态分析和代码样式检查. 该报告类似于构建检查. 修复在构建日志中发现的错误. - -## 功能无状态测试 {#functional-stateless-tests} -为构建在不同配置中的ClickHouse二进制文件运行[无状态功能测试](./tests.md#functional-tests)——发布、调试、使用杀毒软件等.通过报告查看哪些测试失败，然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败.注意, 您必须使用正确的构建配置来重现——在AddressSanitizer下测试可能失败,但在Debug中可以通过.从[CI构建检查页面](./build.md#you-dont-have-to-build-clickhouse)下载二进制文件, 或者在本地构建它. - -## 功能有状态测试 {#functional-stateful-tests} -运行[有状态功能测试](./tests.md#functional-tests).以无状态功能测试相同的方式对待它们.不同之处在于它们需要从[Yandex.Metrica数据集](https://clickhouse.com/docs/en/getting-started/example-datasets/metrica/)的`hits`和`visits`表来运行. - -## 集成测试 {#integration-tests} -运行[集成测试](./tests.md#integration-tests). - -## Testflows 检查{#testflows-check} -使用Testflows测试系统去运行一些测试, 在[此处](../../../tests/testflows/README.md#running-tests-locally)查看如何在本地运行它们. - -## 压力测试 {#stress-test} -从多个客户端并发运行无状态功能测试, 用以检测与并发相关的错误.如果失败: -``` -* Fix all other test failures first; -* Look at the report to find the server logs and check them for possible causes - of error. -``` - -## 冒烟测试 {#split-build-smoke-test} -检查[拆分构建](./build.md#split-build)配置中的服务器构建是否可以启动并运行简单查询.如果失败: -``` -* Fix other test errors first; -* Build the server in [split build](./build.md#split-build) configuration - locally and check whether it can start and run `select 1`. -``` - -## 兼容性检查 {#compatibility-check} -检查`clickhouse`二进制文件是否可以在带有旧libc版本的发行版上运行.如果失败, 请向维护人员寻求帮助. - -## AST模糊器 {#ast-fuzzer} -运行随机生成的查询来捕获程序错误.如果失败, 请向维护人员寻求帮助. - -## 性能测试 {#performance-tests} -测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](../../../docker/test/performance-comparison/README.md#how-to-read-the-report)有详细描述. - -## 质量保证 {#qa} -什么是状态页面上的任务(专用网络)项目? - -它是 Yandex 内部工作系统的链接. Yandex 员工可以看到检查的开始时间及其更详细的状态. - -运行测试的地方 - -Yandex 内部基础设施的某个地方. From 6abe479dd8878432ea270ed9689eaa90fd39cb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 09:33:53 +0800 Subject: [PATCH 090/173] replace symbolic link content. --- docs/zh/development/continuous-integration.md | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 docs/zh/development/continuous-integration.md diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md new file mode 100644 index 00000000000..41e22074915 --- /dev/null +++ b/docs/zh/development/continuous-integration.md @@ -0,0 +1,155 @@ +# 持续集成检查 {#continuous-integration-checks} +当你提交一个pull请求时, ClickHouse[持续集成(CI)系统](../tests.md#test-automation)会对您的代码运行一些自动检查. + +这在存储库维护者(来自ClickHouse团队的人)筛选了您的代码并将可测试标签添加到您的pull请求之后发生. + +检查的结果被列在[GitHub检查文档](https://docs.github.com/en/github/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks)中所述的GitHub pull请求页面. + +如果检查失败，您可能被要求去修复它. 该界面介绍了您可能遇到的检查，以及如何修复它们. + +如果检查失败看起来与您的更改无关, 那么它可能是一些暂时的故障或基础设施问题. 向pull请求推一个空的commit以重新启动CI检查: + +``` +git reset +git commit --allow-empty +git push +``` + +如果您不确定要做什么，可以向维护人员寻求帮助. + +## 与Master合并 {#merge-with-master} +验证PR是否可以合并到master. 如果没有, 它将失败并显示消息'Cannot fetch mergecommit'的.请按[GitHub文档](https://docs.github.com/en/github/collaborating-with-pull-requests/addressing-merge-conflicts/resolving-a-merge-conflict-on-github)中描述的冲突解决, 或使用git将主分支合并到您的pull请求分支来修复这个检查. + +## 文档检查 {#docs-check} +尝试构建ClickHouse文档网站. 如果您更改了文档中的某些内容, 它可能会失败. 最可能的原因是文档中的某些交叉链接是错误的. 转到检查报告并查找`ERROR`和`WARNING`消息. + +### 报告详情 {#report-details} +- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check.html) +- `docs_output.txt`包含构建日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) + +## 描述信息检查 {#description-check} +检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](../../../.github/PULL_REQUEST_TEMPLATE.md)模板. + +您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](../whats-new/changelog/)编写一条用户可读的消息用来描述更改. + +## 推送到DockerHub {#push-to-dockerhub} +生成用于构建和测试的docker映像, 然后将它们推送到DockerHub. + +## 标记检查 {#marker-check} +该检查意味着CI系统已经开始处理PR.当它处于'待处理'状态时，意味着尚未开始所有检查. 启动所有检查后，状态更改为'成功'. + +# 格式检查 {#style-check} +使用`utils/check-style/check-style`二进制文件执行一些简单的基于正则表达式的代码样式检查(注意, 它可以在本地运行). +如果失败, 按照[代码样式指南](./style.md)修复样式错误. + +### 报告详情 {#report-details} +- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check.html) +- `docs_output.txt`记录了查结果错误(无效表格等), 空白页表示没有错误. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/659c78c7abb56141723af6a81bfae39335aa8cb2/style_check/output.txt) + +### PVS 检查 {#pvs-check} +使用静态分析工具[PVS-studio](https://www.viva64.com/en/pvs-studio/)检查代码. 查看报告以查看确切的错误.如果可以则修复它们, 如果不行, 可以向ClickHouse的维护人员寻求帮忙. + +### 报告详情 {#report-details} +- [状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/pvs_check.html) +- `test_run.txt.out.log`包含构建和分析日志文件.它只包含解析或未找到的错误. +- `HTML report`包含分析结果.有关说明请访问PVS的[官方网站](https://www.viva64.com/en/m/0036/#ID14E9A2B2CD) + +## 快速测试 {#fast-test} +通常情况下这是PR运行的第一个检查.它构建ClickHouse以及大多数无状态运行测试, 其中省略了一些.如果失败，在修复之前不会开始进一步的检查. 查看报告以了解哪些测试失败, 然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败. + +### 报告详情 {#report-details} +[状态页示例](https://clickhouse-test-reports.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/fast_test.html) + +#### 状态页文件 {#status-page-files} +- `runlog.out.log` 是包含所有其他日志的通用日志. +- `test_log.txt` +- `submodule_log.txt` 包含关于克隆和检查所需子模块的消息. +- `stderr.log` +- `stdout.log` +- `clickhouse-server.log` +- `clone_log.txt` +- `install_log.txt` +- `clickhouse-server.err.log` +- `build_log.txt` +- `cmake_log.txt` 包含关于C/C++和Linux标志检查的消息. + +#### 状态页列信息 {#status-page-columns} +- 测试名称 -- 包含测试的名称(不带路径, 例如, 所有类型的测试将被剥离到该名称). +- 测试状态 -- 跳过、成功或失败之一. +- 测试时间, 秒. -- 这个测试是空的. + +## 建构检查 {#build-check} +在各种配置中构建ClickHouse, 以便在后续步骤中使用. 您必须修复失败的构建.构建日志通常有足够的信息来修复错误, 但是您可能必须在本地重现故障. `cmake`选项可以在构建日志中通过grep `cmake`操作找到.使用这些选项并遵循[一般的构建过程](./build.md). + +### 报告详情 {#report-details} +[状态页示例](https://clickhouse-builds.s3.yandex.net/12550/67d716b5cc3987801996c31a67b31bf141bc3486/clickhouse_build_check/report.html) +- **Compiler**: `gcc-9` 或 `clang-10` (或其他架构的`clang-10-xx`, 比如`clang-10-freebsd`). +- **Build type**: `Debug` or `RelWithDebInfo` (cmake). +- **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan). +- **Bundled**: `bundled` 构建使用来自 `contrib` 库, 而 `unbundled` 构建使用系统库. +- **Splitted**: `splitted` is a [split build](https://clickhouse.com/docs/en/development/build/#split-build) +- **Status**: `成功` 或 `失败` +- **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用. +- **Build time**. +- **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`). + - `clickhouse-client_XXX_all.deb` + -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb` + - `clickhouse-common-staticXXX_amd64.deb` + - `clickhouse-server_XXX_all.deb` + - `clickhouse-test_XXX_all.deb` + - `clickhouse_XXX_amd64.buildinfo` + - `clickhouse_XXX_amd64.changes` + - `clickhouse`: Main built binary. + - `clickhouse-odbc-bridge` + - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件. + - `shared_build.tgz`: 使用共享库构建. + - `performance.tgz`: 用于性能测试的特殊包. + +## 特殊构建检查 {#special-buildcheck} +使用clang-tidy执行静态分析和代码样式检查. 该报告类似于构建检查. 修复在构建日志中发现的错误. + +## 功能无状态测试 {#functional-stateless-tests} +为构建在不同配置中的ClickHouse二进制文件运行[无状态功能测试](./tests.md#functional-tests)——发布、调试、使用杀毒软件等.通过报告查看哪些测试失败，然后按照[此处](./tests.md#functional-test-locally)描述的在本地重现失败.注意, 您必须使用正确的构建配置来重现——在AddressSanitizer下测试可能失败,但在Debug中可以通过.从[CI构建检查页面](./build.md#you-dont-have-to-build-clickhouse)下载二进制文件, 或者在本地构建它. + +## 功能有状态测试 {#functional-stateful-tests} +运行[有状态功能测试](./tests.md#functional-tests).以无状态功能测试相同的方式对待它们.不同之处在于它们需要从[Yandex.Metrica数据集](https://clickhouse.com/docs/en/getting-started/example-datasets/metrica/)的`hits`和`visits`表来运行. + +## 集成测试 {#integration-tests} +运行[集成测试](./tests.md#integration-tests). + +## Testflows 检查{#testflows-check} +使用Testflows测试系统去运行一些测试, 在[此处](../../../tests/testflows/README.md#running-tests-locally)查看如何在本地运行它们. + +## 压力测试 {#stress-test} +从多个客户端并发运行无状态功能测试, 用以检测与并发相关的错误.如果失败: +``` +* Fix all other test failures first; +* Look at the report to find the server logs and check them for possible causes + of error. +``` + +## 冒烟测试 {#split-build-smoke-test} +检查[拆分构建](./build.md#split-build)配置中的服务器构建是否可以启动并运行简单查询.如果失败: +``` +* Fix other test errors first; +* Build the server in [split build](./build.md#split-build) configuration + locally and check whether it can start and run `select 1`. +``` + +## 兼容性检查 {#compatibility-check} +检查`clickhouse`二进制文件是否可以在带有旧libc版本的发行版上运行.如果失败, 请向维护人员寻求帮助. + +## AST模糊器 {#ast-fuzzer} +运行随机生成的查询来捕获程序错误.如果失败, 请向维护人员寻求帮助. + +## 性能测试 {#performance-tests} +测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](../../../docker/test/performance-comparison/README.md#how-to-read-the-report)有详细描述. + +## 质量保证 {#qa} +什么是状态页面上的任务(专用网络)项目? + +它是 Yandex 内部工作系统的链接. Yandex 员工可以看到检查的开始时间及其更详细的状态. + +运行测试的地方 + +Yandex 内部基础设施的某个地方. From 671ed9d4301211bf49e8629365dc0e0e33347248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 10:23:24 +0800 Subject: [PATCH 091/173] Adjusting invalid links --- docs/zh/development/continuous-integration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh/development/continuous-integration.md b/docs/zh/development/continuous-integration.md index 41e22074915..6cff83067de 100644 --- a/docs/zh/development/continuous-integration.md +++ b/docs/zh/development/continuous-integration.md @@ -1,5 +1,5 @@ # 持续集成检查 {#continuous-integration-checks} -当你提交一个pull请求时, ClickHouse[持续集成(CI)系统](../tests.md#test-automation)会对您的代码运行一些自动检查. +当你提交一个pull请求时, ClickHouse[持续集成(CI)系统](https://clickhouse.com/docs/en/development/tests/#test-automation)会对您的代码运行一些自动检查. 这在存储库维护者(来自ClickHouse团队的人)筛选了您的代码并将可测试标签添加到您的pull请求之后发生. @@ -28,7 +28,7 @@ git push - `docs_output.txt`包含构建日志信息. [成功结果案例](https://clickhouse-test-reports.s3.yandex.net/12550/eabcc293eb02214caa6826b7c15f101643f67a6b/docs_check/docs_output.txt) ## 描述信息检查 {#description-check} -检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](../../../.github/PULL_REQUEST_TEMPLATE.md)模板. +检查pull请求的描述是否符合[PULL_REQUEST_TEMPLATE.md](https://github.com/ClickHouse/ClickHouse/blob/master/.github/PULL_REQUEST_TEMPLATE.md)模板. 您必须为您的更改指定一个更改日志类别(例如，Bug修复), 并且为[CHANGELOG.md](../whats-new/changelog/)编写一条用户可读的消息用来描述更改. @@ -118,7 +118,7 @@ git push 运行[集成测试](./tests.md#integration-tests). ## Testflows 检查{#testflows-check} -使用Testflows测试系统去运行一些测试, 在[此处](../../../tests/testflows/README.md#running-tests-locally)查看如何在本地运行它们. +使用Testflows测试系统去运行一些测试, 在[此处](https://github.com/ClickHouse/ClickHouse/tree/master/tests/testflows#running-tests-locally)查看如何在本地运行它们. ## 压力测试 {#stress-test} 从多个客户端并发运行无状态功能测试, 用以检测与并发相关的错误.如果失败: @@ -143,7 +143,7 @@ git push 运行随机生成的查询来捕获程序错误.如果失败, 请向维护人员寻求帮助. ## 性能测试 {#performance-tests} -测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](../../../docker/test/performance-comparison/README.md#how-to-read-the-report)有详细描述. +测量查询性能的变化. 这是最长的检查, 只需不到 6 小时即可运行.性能测试报告在[此处](https://github.com/ClickHouse/ClickHouse/tree/master/docker/test/performance-comparison#how-to-read-the-report)有详细描述. ## 质量保证 {#qa} 什么是状态页面上的任务(专用网络)项目? From 42c16f5697e79e0d9140202f5b891fe3a838aff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=A8=E6=9C=A8=E5=A4=95120?= Date: Tue, 26 Oct 2021 10:24:38 +0800 Subject: [PATCH 092/173] Adjusting invalid links --- docs/zh/operations/external-authenticators/kerberos.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/external-authenticators/kerberos.md b/docs/zh/operations/external-authenticators/kerberos.md index 1121a86a793..5d2122c5fea 100644 --- a/docs/zh/operations/external-authenticators/kerberos.md +++ b/docs/zh/operations/external-authenticators/kerberos.md @@ -93,7 +93,7 @@ Kerberos主体名称格式通常遵循以下模式: 请注意, 现在, 一旦用户 `my_user` 使用 `kerberos`, 必须在主 `config.xml` 文件中启用 Kerberos，如前所述. ### 使用 SQL 启用 Kerberos {#enabling-kerberos-using-sql} -在 ClickHouse 中启用 [SQL 驱动的访问控制和帐户管理](./access-rights.md#access-control)后, 也可以使用 SQL 语句创建由 Kerberos 识别的用户. +在 ClickHouse 中启用 [SQL 驱动的访问控制和帐户管理](https://clickhouse.com/docs/en/operations/access-rights/#access-control)后, 也可以使用 SQL 语句创建由 Kerberos 识别的用户. ```sql CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' From 97bcceca445cbc1e07e1d2aa9e232610fadac7eb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 Oct 2021 09:49:12 +0300 Subject: [PATCH 093/173] Improve usability of remote_url_allow_hosts --- src/Common/RemoteHostFilter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/RemoteHostFilter.cpp b/src/Common/RemoteHostFilter.cpp index 73c84364f3c..ba7f163fd16 100644 --- a/src/Common/RemoteHostFilter.cpp +++ b/src/Common/RemoteHostFilter.cpp @@ -18,14 +18,14 @@ void RemoteHostFilter::checkURL(const Poco::URI & uri) const { if (!checkForDirectEntry(uri.getHost()) && !checkForDirectEntry(uri.getHost() + ":" + toString(uri.getPort()))) - throw Exception("URL \"" + uri.toString() + "\" is not allowed in config.xml", ErrorCodes::UNACCEPTABLE_URL); + throw Exception("URL \"" + uri.toString() + "\" is not allowed in configuration file, see ", ErrorCodes::UNACCEPTABLE_URL); } void RemoteHostFilter::checkHostAndPort(const std::string & host, const std::string & port) const { if (!checkForDirectEntry(host) && !checkForDirectEntry(host + ":" + port)) - throw Exception("URL \"" + host + ":" + port + "\" is not allowed in config.xml", ErrorCodes::UNACCEPTABLE_URL); + throw Exception("URL \"" + host + ":" + port + "\" is not allowed in configuration file, see ", ErrorCodes::UNACCEPTABLE_URL); } void RemoteHostFilter::setValuesFromConfig(const Poco::Util::AbstractConfiguration & config) From da3e13b730951b87a1ca9087da09ad9fc76e7fa3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 Oct 2021 10:00:19 +0300 Subject: [PATCH 094/173] Fix indecency #30337 --- src/Common/ProgressIndication.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index db1daaac415..26dae8d5b73 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -14,17 +14,19 @@ namespace { - constexpr UInt64 ZERO = 0; + constexpr UInt64 ALL_THREADS = 0; UInt64 calculateNewCoresNumber(DB::ThreadIdToTimeMap const & prev, DB::ThreadIdToTimeMap const& next) { - if (next.find(ZERO) == next.end()) - return ZERO; - auto accumulated = std::accumulate(next.cbegin(), next.cend(), ZERO, - [&prev](UInt64 acc, auto const & elem) + if (next.find(ALL_THREADS) == next.end()) + return 0; + + auto accumulated = std::accumulate(next.cbegin(), next.cend(), 0, + [&prev](UInt64 acc, const auto & elem) { - if (elem.first == ZERO) + if (elem.first == ALL_THREADS) return acc; + auto thread_time = elem.second.time(); auto it = prev.find(elem.first); if (it != prev.end()) @@ -32,9 +34,9 @@ namespace return acc + thread_time; }); - auto elapsed = next.at(ZERO).time() - (prev.contains(ZERO) ? prev.at(ZERO).time() : ZERO); - if (elapsed == ZERO) - return ZERO; + auto elapsed = next.at(ALL_THREADS).time() - (prev.contains(ALL_THREADS) ? prev.at(ALL_THREADS).time() : 0); + if (elapsed == 0) + return 0; return (accumulated + elapsed - 1) / elapsed; } } @@ -109,7 +111,7 @@ size_t ProgressIndication::getUsedThreadsCount() const UInt64 ProgressIndication::getApproximateCoresNumber() const { - return std::accumulate(host_active_cores.cbegin(), host_active_cores.cend(), ZERO, + return std::accumulate(host_active_cores.cbegin(), host_active_cores.cend(), 0, [](UInt64 acc, auto const & elem) { return acc + elem.second; @@ -125,7 +127,7 @@ ProgressIndication::MemoryUsage ProgressIndication::getMemoryUsage() const // In ProfileEvents packets thread id 0 specifies common profiling information // for all threads executing current query on specific host. So instead of summing per thread // memory consumption it's enough to look for data with thread id 0. - if (auto it = host_data.second.find(ZERO); it != host_data.second.end()) + if (auto it = host_data.second.find(ALL_THREADS); it != host_data.second.end()) host_usage = it->second.memory_usage; return MemoryUsage{.total = acc.total + host_usage, .max = std::max(acc.max, host_usage)}; }); From c8f9bab93bcb37b7d30b6dd83d540161f3e8cd5e Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 26 Oct 2021 11:28:28 +0300 Subject: [PATCH 095/173] Better wait for server start in integration tests --- tests/integration/helpers/cluster.py | 29 +++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 3854cadaba5..c7b140b6471 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -2056,12 +2056,31 @@ class ClickHouseInstance: def start_clickhouse(self, start_wait_sec=30): if not self.stay_alive: - raise Exception("clickhouse can be started again only with stay_alive=True instance") + raise Exception("ClickHouse can be started again only with stay_alive=True instance") + + time_to_sleep = 0.5 + start_tries = 5 + + total_tries = int(start_wait_sec / time_to_sleep) + query_tries = int(total_tries / start_tries) + + for i in range(start_tries): + # sometimes after SIGKILL (hard reset) server may refuse to start for some time + # for different reasons + self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) + started = False + for _ in range(query_tries): + try: + self.query("select 1") + started = True + break + except: + time.sleep(time_to_sleep) + if started: + break + else: + raise Exception("Cannot start ClickHouse, se additional info in logs") - self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) - # wait start - from helpers.test_tools import assert_eq_with_retry - assert_eq_with_retry(self, "select 1", "1", retry_count=int(start_wait_sec / 0.5), sleep_time=0.5) def restart_clickhouse(self, stop_start_wait_sec=30, kill=False): self.stop_clickhouse(stop_start_wait_sec, kill) From b7ed729c81f904b70a7d71c8498d3bead4b823e3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 26 Oct 2021 11:29:22 +0300 Subject: [PATCH 096/173] typo --- tests/integration/helpers/cluster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index c7b140b6471..9d7657d3a83 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -2066,7 +2066,7 @@ class ClickHouseInstance: for i in range(start_tries): # sometimes after SIGKILL (hard reset) server may refuse to start for some time - # for different reasons + # for different reasons. self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) started = False for _ in range(query_tries): @@ -2079,7 +2079,7 @@ class ClickHouseInstance: if started: break else: - raise Exception("Cannot start ClickHouse, se additional info in logs") + raise Exception("Cannot start ClickHouse, see additional info in logs") def restart_clickhouse(self, stop_start_wait_sec=30, kill=False): From 3b5b1d20061304c65049cdb67c5b9e473d224440 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 26 Oct 2021 12:07:07 +0300 Subject: [PATCH 097/173] Fix keeper zookeeper converter test --- .../test_keeper_zookeeper_converter/configs/keeper_config.xml | 4 ++-- tests/integration/test_keeper_zookeeper_converter/test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_keeper_zookeeper_converter/configs/keeper_config.xml b/tests/integration/test_keeper_zookeeper_converter/configs/keeper_config.xml index d98c8d72e61..612cb979bb7 100644 --- a/tests/integration/test_keeper_zookeeper_converter/configs/keeper_config.xml +++ b/tests/integration/test_keeper_zookeeper_converter/configs/keeper_config.xml @@ -6,8 +6,8 @@ /var/lib/clickhouse/coordination/snapshots - 5000 - 10000 + 30000 + 600000 trace 75 diff --git a/tests/integration/test_keeper_zookeeper_converter/test.py b/tests/integration/test_keeper_zookeeper_converter/test.py index eac2b4c45c5..6829b4a9000 100644 --- a/tests/integration/test_keeper_zookeeper_converter/test.py +++ b/tests/integration/test_keeper_zookeeper_converter/test.py @@ -60,12 +60,12 @@ def started_cluster(): finally: cluster.shutdown() -def get_fake_zk(timeout=30.0): +def get_fake_zk(timeout=60.0): _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip('node') + ":9181", timeout=timeout) _fake_zk_instance.start() return _fake_zk_instance -def get_genuine_zk(timeout=30.0): +def get_genuine_zk(timeout=60.0): _genuine_zk_instance = KazooClient(hosts=cluster.get_instance_ip('node') + ":2181", timeout=timeout) _genuine_zk_instance.start() return _genuine_zk_instance From 73bbf7549af0e5f11c8f0b858f466fadc9b15502 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 21 Oct 2021 17:13:39 +0300 Subject: [PATCH 098/173] HashedArrayDictionary optimize read multiple attributes --- src/Dictionaries/HashedArrayDictionary.cpp | 210 +++++++++++++++++++++ src/Dictionaries/HashedArrayDictionary.h | 22 +++ 2 files changed, 232 insertions(+) diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 426857e055f..0ba73b8bd72 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -146,6 +146,85 @@ ColumnPtr HashedArrayDictionary::getColumn( return result; } +template +Columns HashedArrayDictionary::getColumns( + const Strings & attribute_names, + const DataTypes & result_types, + const Columns & key_columns, + const DataTypes & key_types, + const Columns & default_values_columns) const +{ + if (dictionary_key_type == DictionaryKeyType::Complex) + dict_struct.validateKeyTypes(key_types); + + DictionaryKeysArenaHolder arena_holder; + DictionaryKeysExtractor extractor(key_columns, arena_holder.getComplexKeyArena()); + + const size_t keys_size = extractor.getKeysSize(); + + PaddedPODArray key_index_to_element_index; + + /** Optimization for multiple attributes. + * For each key save element index in key_index_to_element_index array. + * Later in type_call for attribute use getItemsImpl specialization with key_index_to_element_index array + * intead of DictionaryKeyExtractor. + */ + if (attribute_names.size() > 1) { + const auto & key_attribute_container = key_attribute.container; + size_t keys_found = 0; + + key_index_to_element_index.resize(keys_size); + + for (size_t key_index = 0; key_index < keys_size; ++key_index) + { + auto key = extractor.extractCurrentKey(); + + auto it = key_attribute_container.find(key); + if (it == key_attribute_container.end()) + { + key_index_to_element_index[key_index] = -1; + } + else + { + key_index_to_element_index[key_index] = it->getMapped(); + ++keys_found; + } + + extractor.rollbackCurrentKey(); + } + + query_count.fetch_add(keys_size, std::memory_order_relaxed); + found_count.fetch_add(keys_found, std::memory_order_relaxed); + } + + size_t attribute_names_size = attribute_names.size(); + + Columns result_columns; + result_columns.reserve(attribute_names_size); + + for (size_t i = 0; i < attribute_names_size; ++i) + { + ColumnPtr result_column; + + const auto & attribute_name = attribute_names[i]; + const auto & result_type = result_types[i]; + const auto & default_values_column = default_values_columns[i]; + + const auto & dictionary_attribute = dict_struct.getAttribute(attribute_name, result_type); + const size_t attribute_index = dict_struct.attribute_name_to_index.find(attribute_name)->second; + auto & attribute = attributes[attribute_index]; + + if (attribute_names_size > 1) + result_column = getAttributeColumn(attribute, dictionary_attribute, keys_size, default_values_column, key_index_to_element_index); + else + result_column = getAttributeColumn(attribute, dictionary_attribute, keys_size, default_values_column, extractor); + + result_columns.emplace_back(std::move(result_column)); + } + + return result_columns; +} + template ColumnUInt8::Ptr HashedArrayDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const { @@ -499,6 +578,102 @@ void HashedArrayDictionary::resize(size_t added_rows) key_attribute.container.reserve(added_rows); } +template +template +ColumnPtr HashedArrayDictionary::getAttributeColumn( + const Attribute & attribute, + const DictionaryAttribute & dictionary_attribute, + size_t keys_size, + ColumnPtr default_values_column, + KeysProvider && keys_object) const +{ + ColumnPtr result; + + bool is_attribute_nullable = attribute.is_index_null.has_value(); + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to = nullptr; + if (attribute.is_index_null) + { + col_null_map_to = ColumnUInt8::create(keys_size, false); + vec_null_map_to = &col_null_map_to->getData(); + } + + auto type_call = [&](const auto & dictionary_attribute_type) + { + using Type = std::decay_t; + using AttributeType = typename Type::AttributeType; + using ValueType = DictionaryValueType; + using ColumnProvider = DictionaryAttributeColumnProvider; + + DictionaryDefaultValueExtractor default_value_extractor(dictionary_attribute.null_value, default_values_column); + + auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size); + + if constexpr (std::is_same_v) + { + auto * out = column.get(); + + getItemsImpl( + attribute, + keys_object, + [&](const size_t, const Array & value, bool) { out->insert(value); }, + default_value_extractor); + } + else if constexpr (std::is_same_v) + { + auto * out = column.get(); + + if (is_attribute_nullable) + getItemsImpl( + attribute, + keys_object, + [&](size_t row, const StringRef value, bool is_null) + { + (*vec_null_map_to)[row] = is_null; + out->insertData(value.data, value.size); + }, + default_value_extractor); + else + getItemsImpl( + attribute, + keys_object, + [&](size_t, const StringRef value, bool) { out->insertData(value.data, value.size); }, + default_value_extractor); + } + else + { + auto & out = column->getData(); + + if (is_attribute_nullable) + getItemsImpl( + attribute, + keys_object, + [&](size_t row, const auto value, bool is_null) + { + (*vec_null_map_to)[row] = is_null; + out[row] = value; + }, + default_value_extractor); + else + getItemsImpl( + attribute, + keys_object, + [&](size_t row, const auto value, bool) { out[row] = value; }, + default_value_extractor); + } + + result = std::move(column); + }; + + callOnDictionaryAttributeType(attribute.type, type_call); + + if (is_attribute_nullable) + result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + + return result; +} + template template void HashedArrayDictionary::getItemsImpl( @@ -547,6 +722,41 @@ void HashedArrayDictionary::getItemsImpl( found_count.fetch_add(keys_found, std::memory_order_relaxed); } +template +template +void HashedArrayDictionary::getItemsImpl( + const Attribute & attribute, + const PaddedPODArray & key_index_to_element_index, + ValueSetter && set_value, + DefaultValueExtractor & default_value_extractor) const +{ + const auto & attribute_container = std::get>(attribute.container); + const size_t keys_size = key_index_to_element_index.size(); + + for (size_t key_index = 0; key_index < keys_size; ++key_index) + { + bool key_exists = key_index_to_element_index[key_index] != -1; + + if (key_exists) + { + size_t element_index = static_cast(key_index_to_element_index[key_index]); + const auto & element = attribute_container[element_index]; + + if constexpr (is_nullable) + set_value(key_index, element, (*attribute.is_index_null)[element_index]); + else + set_value(key_index, element, false); + } + else + { + if constexpr (is_nullable) + set_value(key_index, default_value_extractor[key_index], default_value_extractor.isNullAt(key_index)); + else + set_value(key_index, default_value_extractor[key_index], false); + } + } +} + template StringRef HashedArrayDictionary::copyKeyInArena(StringRef key) { diff --git a/src/Dictionaries/HashedArrayDictionary.h b/src/Dictionaries/HashedArrayDictionary.h index 0af3fb65581..5ad1efeb056 100644 --- a/src/Dictionaries/HashedArrayDictionary.h +++ b/src/Dictionaries/HashedArrayDictionary.h @@ -93,6 +93,13 @@ public: const DataTypes & key_types, const ColumnPtr & default_values_column) const override; + Columns getColumns( + const Strings & attribute_names, + const DataTypes & result_types, + const Columns & key_columns, + const DataTypes & key_types, + const Columns & default_values_columns) const override; + ColumnUInt8::Ptr hasKeys(const Columns & key_columns, const DataTypes & key_types) const override; bool hasHierarchy() const override { return dictionary_key_type == DictionaryKeyType::Simple && dict_struct.hierarchical_attribute_index.has_value(); } @@ -170,6 +177,14 @@ private: void calculateBytesAllocated(); + template + ColumnPtr getAttributeColumn( + const Attribute & attribute, + const DictionaryAttribute & dictionary_attribute, + size_t keys_size, + ColumnPtr default_values_column, + KeysProvider && keys_object) const; + template void getItemsImpl( const Attribute & attribute, @@ -177,6 +192,13 @@ private: ValueSetter && set_value, DefaultValueExtractor & default_value_extractor) const; + template + void getItemsImpl( + const Attribute & attribute, + const PaddedPODArray & key_index_to_element_index, + ValueSetter && set_value, + DefaultValueExtractor & default_value_extractor) const; + template void getAttributeContainer(size_t attribute_index, GetContainerFunc && get_container_func); From 6cf16862abae23bc25366b78b6f841d3abec3a44 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 22 Oct 2021 11:03:40 +0300 Subject: [PATCH 099/173] Fixed style check --- src/Dictionaries/HashedArrayDictionary.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 0ba73b8bd72..94145e8fa99 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -167,9 +167,10 @@ Columns HashedArrayDictionary::getColumns( /** Optimization for multiple attributes. * For each key save element index in key_index_to_element_index array. * Later in type_call for attribute use getItemsImpl specialization with key_index_to_element_index array - * intead of DictionaryKeyExtractor. + * instead of DictionaryKeyExtractor. */ - if (attribute_names.size() > 1) { + if (attribute_names.size() > 1) + { const auto & key_attribute_container = key_attribute.container; size_t keys_found = 0; From dbf481102986780667fac1da04b91ae8258450cd Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 26 Oct 2021 13:00:42 +0300 Subject: [PATCH 100/173] Updated documentation --- src/Common/LRUCache.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Common/LRUCache.h b/src/Common/LRUCache.h index 3df27cbdce9..bbc09fd3aff 100644 --- a/src/Common/LRUCache.h +++ b/src/Common/LRUCache.h @@ -36,6 +36,9 @@ public: using Mapped = TMapped; using MappedPtr = std::shared_ptr; + /** Initialize LRUCache with max_size and max_elements_size. + * max_elements_size == 0 means no elements size restrictions. + */ LRUCache(size_t max_size_, size_t max_elements_size_ = 0) : max_size(std::max(static_cast(1), max_size_)) , max_elements_size(max_elements_size_) From a1cab43feb52936e6969fa7d119733efe7808407 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 26 Oct 2021 13:32:07 +0300 Subject: [PATCH 101/173] fix five years old bug in StingSearcher --- src/Common/StringSearcher.h | 133 ++++++------------ ...163_search_case_insensetive_utf8.reference | 12 ++ .../01163_search_case_insensetive_utf8.sql | 12 ++ 3 files changed, 66 insertions(+), 91 deletions(-) create mode 100644 tests/queries/0_stateless/01163_search_case_insensetive_utf8.reference create mode 100644 tests/queries/0_stateless/01163_search_case_insensetive_utf8.sql diff --git a/src/Common/StringSearcher.h b/src/Common/StringSearcher.h index af1d36adf81..f34bc6f7322 100644 --- a/src/Common/StringSearcher.h +++ b/src/Common/StringSearcher.h @@ -116,9 +116,9 @@ public: /// lower and uppercase variants of the first octet of the first character in `needle` size_t length_l = UTF8::convertCodePointToUTF8(first_l_u32, l_seq, sizeof(l_seq)); - size_t length_r = UTF8::convertCodePointToUTF8(first_u_u32, u_seq, sizeof(u_seq)); + size_t length_u = UTF8::convertCodePointToUTF8(first_u_u32, u_seq, sizeof(u_seq)); - if (length_l != length_r) + if (length_l != length_u) throw Exception{"UTF8 sequences with different lowercase and uppercase lengths are not supported", ErrorCodes::UNSUPPORTED_PARAMETER}; } @@ -183,6 +183,31 @@ public: #endif } + template > + ALWAYS_INLINE bool compareTrivial(const CharT * haystack_pos, const CharT * const haystack_end, const uint8_t * needle_pos) const + { + while (haystack_pos < haystack_end && needle_pos < needle_end) + { + auto haystack_code_point = UTF8::convertUTF8ToCodePoint(haystack_pos, haystack_end - haystack_pos); + auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos); + + /// Invalid UTF-8, should not compare equals + if (!haystack_code_point || !needle_code_point) + break; + + /// Not equals case insensitive. + if (Poco::Unicode::toLower(*haystack_code_point) != Poco::Unicode::toLower(*needle_code_point)) + break; + + /// @note assuming sequences for lowercase and uppercase have exact same length (that is not always true) + const auto len = UTF8::seqLength(*haystack_pos); + haystack_pos += len; + needle_pos += len; + } + + return needle_pos == needle_end; + } + template > ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * haystack_end, const CharT * pos) const { @@ -200,34 +225,15 @@ public: { if (mask == cachemask) { - pos += cache_valid_len; - auto needle_pos = needle + cache_valid_len; - - while (needle_pos < needle_end) - { - auto haystack_code_point = UTF8::convertUTF8ToCodePoint(pos, haystack_end - pos); - auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos); - - /// Invalid UTF-8, should not compare equals - if (!haystack_code_point || !needle_code_point) - break; - - /// Not equals case insensitive. - if (Poco::Unicode::toLower(*haystack_code_point) != Poco::Unicode::toLower(*needle_code_point)) - break; - - /// @note assuming sequences for lowercase and uppercase have exact same length (that is not always true) - const auto len = UTF8::seqLength(*pos); - pos += len; - needle_pos += len; - } - - if (needle_pos == needle_end) + if (compareTrivial(pos, haystack_end, needle)) return true; } } else if ((mask & cachemask) == cachemask) - return true; + { + if (compareTrivial(pos, haystack_end, needle)) + return true; + } return false; } @@ -238,25 +244,7 @@ public: pos += first_needle_symbol_is_ascii; auto needle_pos = needle + first_needle_symbol_is_ascii; - while (needle_pos < needle_end) - { - auto haystack_code_point = UTF8::convertUTF8ToCodePoint(pos, haystack_end - pos); - auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos); - - /// Invalid UTF-8, should not compare equals - if (!haystack_code_point || !needle_code_point) - break; - - /// Not equals case insensitive. - if (Poco::Unicode::toLower(*haystack_code_point) != Poco::Unicode::toLower(*needle_code_point)) - break; - - const auto len = UTF8::seqLength(*pos); - pos += len; - needle_pos += len; - } - - if (needle_pos == needle_end) + if (compareTrivial(pos, haystack_end, needle_pos)) return true; } @@ -299,40 +287,21 @@ public: const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel); const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu); const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset); - const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset); + const auto mask_offset_both = _mm_movemask_epi8(v_against_l_or_u_offset); if (0xffff == cachemask) { - if (mask_offset == cachemask) + if (mask_offset_both == cachemask) { - auto haystack_pos = haystack + cache_valid_len; - auto needle_pos = needle + cache_valid_len; - - while (haystack_pos < haystack_end && needle_pos < needle_end) - { - auto haystack_code_point = UTF8::convertUTF8ToCodePoint(haystack_pos, haystack_end - haystack_pos); - auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos); - - /// Invalid UTF-8, should not compare equals - if (!haystack_code_point || !needle_code_point) - break; - - /// Not equals case insensitive. - if (Poco::Unicode::toLower(*haystack_code_point) != Poco::Unicode::toLower(*needle_code_point)) - break; - - /// @note assuming sequences for lowercase and uppercase have exact same length (that is not always true) - const auto len = UTF8::seqLength(*haystack_pos); - haystack_pos += len; - needle_pos += len; - } - - if (needle_pos == needle_end) + if (compareTrivial(haystack, haystack_end, needle)) return haystack; } } - else if ((mask_offset & cachemask) == cachemask) - return haystack; + else if ((mask_offset_both & cachemask) == cachemask) + { + if (compareTrivial(haystack, haystack_end, needle)) + return haystack; + } /// first octet was ok, but not the first 16, move to start of next sequence and reapply haystack += UTF8::seqLength(*haystack); @@ -349,25 +318,7 @@ public: auto haystack_pos = haystack + first_needle_symbol_is_ascii; auto needle_pos = needle + first_needle_symbol_is_ascii; - while (haystack_pos < haystack_end && needle_pos < needle_end) - { - auto haystack_code_point = UTF8::convertUTF8ToCodePoint(haystack_pos, haystack_end - haystack_pos); - auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos); - - /// Invalid UTF-8, should not compare equals - if (!haystack_code_point || !needle_code_point) - break; - - /// Not equals case insensitive. - if (Poco::Unicode::toLower(*haystack_code_point) != Poco::Unicode::toLower(*needle_code_point)) - break; - - const auto len = UTF8::seqLength(*haystack_pos); - haystack_pos += len; - needle_pos += len; - } - - if (needle_pos == needle_end) + if (compareTrivial(haystack_pos, haystack_end, needle_pos)) return haystack; } diff --git a/tests/queries/0_stateless/01163_search_case_insensetive_utf8.reference b/tests/queries/0_stateless/01163_search_case_insensetive_utf8.reference new file mode 100644 index 00000000000..66f4ca4a5a8 --- /dev/null +++ b/tests/queries/0_stateless/01163_search_case_insensetive_utf8.reference @@ -0,0 +1,12 @@ +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/01163_search_case_insensetive_utf8.sql b/tests/queries/0_stateless/01163_search_case_insensetive_utf8.sql new file mode 100644 index 00000000000..99bdd38ceae --- /dev/null +++ b/tests/queries/0_stateless/01163_search_case_insensetive_utf8.sql @@ -0,0 +1,12 @@ +SELECT positionCaseInsensitiveUTF8(materialize('сссссс'), 'Ё'); +SELECT countSubstringsCaseInsensitiveUTF8(materialize('сссссс'), 'ё'); +SELECT positionCaseInsensitiveUTF8(materialize('сссссссс'), 'ё'); +SELECT countSubstringsCaseInsensitiveUTF8(materialize('сссссссс'), 'Ё'); +SELECT countSubstringsCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ёёёёёёё'); +SELECT positionCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ёЁёЁёЁё'); +SELECT countSubstringsCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ёЁёЁёЁёЁёЁ'); +SELECT positionCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ЁЁЁЁЁЁЁЁЁЁ'); +SELECT countSubstringsCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ёЁёЁёЁёссс'); +SELECT positionCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ёЁёЁёЁёссс'); +SELECT countSubstringsCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ЁС'); +SELECT positionCaseInsensitiveUTF8(materialize('ссссссссссссссссссс'), 'ёс'); From 2e7e195e776dbfeb3c5832852ee277187e4e50db Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 25 Oct 2021 20:49:49 +0300 Subject: [PATCH 102/173] change alter_lock to std::timed_mutex --- src/Interpreters/InterpreterAlterQuery.cpp | 3 ++- src/Storages/IStorage.cpp | 18 +++++++++++------- src/Storages/IStorage.h | 14 ++++++-------- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.h | 2 +- src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageBuffer.h | 2 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageDistributed.h | 2 +- src/Storages/StorageMaterializedView.cpp | 2 +- src/Storages/StorageMaterializedView.h | 2 +- src/Storages/StorageMerge.cpp | 2 +- src/Storages/StorageMerge.h | 2 +- src/Storages/StorageMergeTree.cpp | 2 +- src/Storages/StorageMergeTree.h | 2 +- src/Storages/StorageNull.cpp | 2 +- src/Storages/StorageNull.h | 2 +- src/Storages/StorageProxy.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 7 ++++--- src/Storages/StorageReplicatedMergeTree.h | 2 +- src/Storages/TableLockHolder.h | 2 -- 21 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index ace82f47f05..8e84373cb0f 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -76,7 +76,8 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); if (table->isStaticStorage()) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); - auto alter_lock = table->lockForAlter(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Add default database to table identifiers that we can encounter in e.g. default expressions, mutation expression, etc. diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 7c158794caf..dd851f19906 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -60,27 +60,31 @@ TableLockHolder IStorage::lockForShare(const String & query_id, const std::chron return result; } -TableLockHolder IStorage::lockForAlter(const String & query_id, const std::chrono::milliseconds & acquire_timeout) +IStorage::AlterLockHolder IStorage::lockForAlter(const std::chrono::milliseconds & acquire_timeout) { - TableLockHolder result = tryLockTimed(alter_lock, RWLockImpl::Write, query_id, acquire_timeout); + AlterLockHolder lock{alter_lock, std::defer_lock}; + + if (!lock.try_lock_for(acquire_timeout)) + throw Exception(ErrorCodes::DEADLOCK_AVOIDED, + "Locking attempt for ALTER on \"{}\" has timed out! ({} ms) " + "Possible deadlock avoided. Client should retry.", + getStorageID().getFullTableName(), std::to_string(acquire_timeout.count())); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); - return result; + return lock; } TableExclusiveLockHolder IStorage::lockExclusively(const String & query_id, const std::chrono::milliseconds & acquire_timeout) { TableExclusiveLockHolder result; - result.alter_lock = tryLockTimed(alter_lock, RWLockImpl::Write, query_id, acquire_timeout); + result.drop_lock = tryLockTimed(drop_lock, RWLockImpl::Write, query_id, acquire_timeout); if (is_dropped) throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED); - result.drop_lock = tryLockTimed(drop_lock, RWLockImpl::Write, query_id, acquire_timeout); - return result; } @@ -126,7 +130,7 @@ Pipe IStorage::alterPartition( throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } -void IStorage::alter(const AlterCommands & params, ContextPtr context, TableLockHolder &) +void IStorage::alter(const AlterCommands & params, ContextPtr context, AlterLockHolder &) { auto table_id = getStorageID(); StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 701cf127521..aaaf919d3bf 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -238,7 +238,8 @@ public: /// Lock table for alter. This lock must be acuqired in ALTER queries to be /// sure, that we execute only one simultaneous alter. Doesn't affect share lock. - TableLockHolder lockForAlter(const String & query_id, const std::chrono::milliseconds & acquire_timeout); + using AlterLockHolder = std::unique_lock; + AlterLockHolder lockForAlter(const std::chrono::milliseconds & acquire_timeout); /// Lock table exclusively. This lock must be acquired if you want to be /// sure, that no other thread (SELECT, merge, ALTER, etc.) doing something @@ -417,7 +418,7 @@ public: /** ALTER tables in the form of column changes that do not affect the change * to Storage or its parameters. Executes under alter lock (lockForAlter). */ - virtual void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & alter_lock_holder); + virtual void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & alter_lock_holder); /** Checks that alter commands can be applied to storage. For example, columns can be modified, * or primary key can be changes, etc. @@ -590,12 +591,9 @@ public: virtual bool dropTableImmediately() { return false; } private: - /// Lock required for alter queries (lockForAlter). Always taken for write - /// (actually can be replaced with std::mutex, but for consistency we use - /// RWLock). Allows to execute only one simultaneous alter query. Also it - /// should be taken by DROP-like queries, to be sure, that all alters are - /// finished. - mutable RWLock alter_lock = RWLockImpl::create(); + /// Lock required for alter queries (lockForAlter). + /// Allows to execute only one simultaneous alter query. + mutable std::timed_mutex alter_lock; /// Lock required for drop queries. Every thread that want to ensure, that /// table is not dropped have to table this lock for read (lockForShare). diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 57729ef2eda..235ad79064a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2211,7 +2211,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( void MergeTreeData::changeSettings( const ASTPtr & new_settings, - TableLockHolder & /* table_lock_holder */) + AlterLockHolder & /* table_lock_holder */) { if (new_settings) { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 2ea6a89002c..22ec7ce6f53 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -583,7 +583,7 @@ public: /// Change MergeTreeSettings void changeSettings( const ASTPtr & new_settings, - TableLockHolder & table_lock_holder); + AlterLockHolder & table_lock_holder); /// Should be called if part data is suspected to be corrupted. void reportBrokenPart(const String & name) const diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index a74223054e9..7d886551a0e 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -1058,7 +1058,7 @@ std::optional StorageBuffer::totalBytes(const Settings & /*settings*/) c return total_writes.bytes; } -void StorageBuffer::alter(const AlterCommands & params, ContextPtr local_context, TableLockHolder &) +void StorageBuffer::alter(const AlterCommands & params, ContextPtr local_context, AlterLockHolder &) { auto table_id = getStorageID(); checkAlterIsPossible(params, local_context); diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 3e8955ad864..28f8de5fd88 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -107,7 +107,7 @@ public: void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; /// The structure of the subordinate table is not checked and does not change. - void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & table_lock_holder) override; std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 1dd75f09b92..2c528a4ebf5 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -785,7 +785,7 @@ void StorageDistributed::checkAlterIsPossible(const AlterCommands & commands, Co } } -void StorageDistributed::alter(const AlterCommands & params, ContextPtr local_context, TableLockHolder &) +void StorageDistributed::alter(const AlterCommands & params, ContextPtr local_context, AlterLockHolder &) { auto table_id = getStorageID(); diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 50e9c935a17..540ae413f19 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -94,7 +94,7 @@ public: /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked - void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & table_lock_holder) override; void startup() override; void shutdown() override; diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 29dbd6d38ed..361ff95984f 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -274,7 +274,7 @@ bool StorageMaterializedView::optimize( void StorageMaterializedView::alter( const AlterCommands & params, ContextPtr local_context, - TableLockHolder &) + AlterLockHolder &) { auto table_id = getStorageID(); StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index b6f677e84bb..105f3f25533 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -50,7 +50,7 @@ public: const Names & deduplicate_by_columns, ContextPtr context) override; - void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & table_lock_holder) override; void checkMutationIsPossible(const MutationCommands & commands, const Settings & settings) const override; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 38db0b61e8d..979e027d338 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -651,7 +651,7 @@ void StorageMerge::checkAlterIsPossible(const AlterCommands & commands, ContextP } void StorageMerge::alter( - const AlterCommands & params, ContextPtr local_context, TableLockHolder &) + const AlterCommands & params, ContextPtr local_context, AlterLockHolder &) { auto table_id = getStorageID(); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index b9a127515f8..52a91747433 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -43,7 +43,7 @@ public: /// you need to add and remove columns in the sub-tables manually /// the structure of sub-tables is not checked - void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & table_lock_holder) override; bool mayBenefitFromIndexForIn( const ASTPtr & left_in_operand, ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot) const override; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 699c78cdab9..84022e3278d 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -271,7 +271,7 @@ void StorageMergeTree::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont void StorageMergeTree::alter( const AlterCommands & commands, ContextPtr local_context, - TableLockHolder & table_lock_holder) + AlterLockHolder & table_lock_holder) { auto table_id = getStorageID(); auto old_storage_settings = getSettings(); diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index db651bd00c3..8ed4b707b34 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -86,7 +86,7 @@ public: void drop() override; void truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) override; - void alter(const AlterCommands & commands, ContextPtr context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & commands, ContextPtr context, AlterLockHolder & table_lock_holder) override; void checkTableCanBeDropped() const override; diff --git a/src/Storages/StorageNull.cpp b/src/Storages/StorageNull.cpp index 2b3585e360e..094e345e433 100644 --- a/src/Storages/StorageNull.cpp +++ b/src/Storages/StorageNull.cpp @@ -64,7 +64,7 @@ void StorageNull::checkAlterIsPossible(const AlterCommands & commands, ContextPt } -void StorageNull::alter(const AlterCommands & params, ContextPtr context, TableLockHolder &) +void StorageNull::alter(const AlterCommands & params, ContextPtr context, AlterLockHolder &) { auto table_id = getStorageID(); diff --git a/src/Storages/StorageNull.h b/src/Storages/StorageNull.h index de667c1d75c..5fef7f984e4 100644 --- a/src/Storages/StorageNull.h +++ b/src/Storages/StorageNull.h @@ -43,7 +43,7 @@ public: void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; - void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & table_lock_holder) override; std::optional totalRows(const Settings &) const override { diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 62e0ccc76aa..304f84c02eb 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -91,7 +91,7 @@ public: IStorage::renameInMemory(new_table_id); } - void alter(const AlterCommands & params, ContextPtr context, TableLockHolder & alter_lock_holder) override + void alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & alter_lock_holder) override { getNested()->alter(params, context, alter_lock_holder); IStorage::setInMemoryMetadata(getNested()->getInMemoryMetadata()); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ef8052de0ec..4e2cad668a1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4355,7 +4355,8 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer zookeeper->multi(requests); { - auto lock = lockForAlter(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); + auto table_lock_holder = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); + auto alter_lock_holder = lockForAlter(getSettings()->lock_acquire_timeout_for_background_operations); LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry); @@ -4427,7 +4428,7 @@ PartitionBlockNumbersHolder StorageReplicatedMergeTree::allocateBlockNumbersInAf void StorageReplicatedMergeTree::alter( - const AlterCommands & commands, ContextPtr query_context, TableLockHolder & table_lock_holder) + const AlterCommands & commands, ContextPtr query_context, AlterLockHolder & table_lock_holder) { assertNotReadonly(); @@ -4632,7 +4633,7 @@ void StorageReplicatedMergeTree::alter( } } - table_lock_holder.reset(); + table_lock_holder.unlock(); LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); waitForLogEntryToBeProcessedIfNecessary(*alter_entry, query_context, "Some replicas doesn't finish metadata alter: "); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index bc03dbb45ae..d3d8463bffd 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -129,7 +129,7 @@ public: const Names & deduplicate_by_columns, ContextPtr query_context) override; - void alter(const AlterCommands & commands, ContextPtr query_context, TableLockHolder & table_lock_holder) override; + void alter(const AlterCommands & commands, ContextPtr query_context, AlterLockHolder & table_lock_holder) override; void mutate(const MutationCommands & commands, ContextPtr context) override; void waitMutation(const String & znode_name, size_t mutations_sync) const; diff --git a/src/Storages/TableLockHolder.h b/src/Storages/TableLockHolder.h index 6ec06a5e66c..36fda2c69bb 100644 --- a/src/Storages/TableLockHolder.h +++ b/src/Storages/TableLockHolder.h @@ -16,8 +16,6 @@ struct TableExclusiveLockHolder private: friend class IStorage; - /// Order is important. - TableLockHolder alter_lock; TableLockHolder drop_lock; }; From 2d559e098c4246726f5850d204e1bf5a257aefb2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 26 Oct 2021 14:06:09 +0300 Subject: [PATCH 103/173] Fix test --- .../queries/0_stateless/00652_replicated_mutations_zookeeper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh index d0e40e2ee62..f070e34f2a1 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh @@ -63,7 +63,7 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r2(x UInt32) ENGINE cleanup_delay_period_random_add = 0" # Insert some data -${CLICKHOUSE_CLIENT} --query="INSERT INTO mutations_cleaner_r1(x) VALUES (1), (2), (3), (4)" +${CLICKHOUSE_CLIENT} --query="INSERT INTO mutations_cleaner_r1(x) VALUES (1), (2), (3), (4), (5)" # Add some mutations and wait for their execution ${CLICKHOUSE_CLIENT} --query="ALTER TABLE mutations_cleaner_r1 DELETE WHERE x = 1 SETTINGS mutations_sync = 2" From 3b5c4b7fd269b35f8c6840fc59400dce6d796b0a Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Tue, 26 Oct 2021 16:33:58 +0300 Subject: [PATCH 104/173] Apply suggestions from code review Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/operations/settings/settings.md | 4 ++-- docs/ru/operations/settings/settings.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 47e0a3006da..34053ae247a 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3849,8 +3849,8 @@ Enables or disables waiting for processing of asynchronous insertion. If enabled Possible values: -- 0 — Disabled. -- 1 — Enabled. +- 0 — Client returns `OK` even if the data is not yet inserted. +- 1 — Client returns `OK` only after the data is inserted. Default value: `1`. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index fb84913c0db..826bd6db77d 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3645,9 +3645,9 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Включает или отключает асинхронные вставки для нескольких параллельных соединений. Работает только для вставок по протоколу HTTP. Обратите внимание, что при таких вставках дедупликация не производится. -Если включено, данные собираются в пачки перед вставкой в таблицу. Это позволяет производить мелкие и частые вставки в Clickhouse (до 15000 запросов в секунду) без промежуточных таблиц. +Если включено, данные собираются в пачки перед вставкой в таблицу. Это позволяет производить мелкие и частые вставки в ClickHouse (до 15000 запросов в секунду) без промежуточных таблиц. -Вставка данных происходит либо как только привышен [async_insert_max_data_size](#async-insert-max-data-size), либо через [async_insert_busy_timeout](#async-insert-busy-timeout) миллисекунд после первого запроса `INSERT`. Если [async_insert_stale_timeout](#async-insert-stale-timeout) задано ненулевое значение, то данные вставляются через `async_insert_stale_timeout` после последнего запроса. +Вставка данных происходит либо как только объем вставляемых данных превышает [async_insert_max_data_size](#async-insert-max-data-size), либо через [async_insert_busy_timeout](#async-insert-busy-timeout) миллисекунд после первого запроса `INSERT`. Если в [async_insert_stale_timeout](#async-insert-stale-timeout) задано ненулевое значение, то данные вставляются через `async_insert_stale_timeout` миллисекунд после последнего запроса. Если включен параметр [wait_for_async_insert](#wait-for-async-insert), каждый клиент ждет, пока данные будут сброшены в таблицу. Иначе запрос будет обработан почти моментально, даже если данные еще не вставлены. @@ -3664,7 +3664,7 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Возможные значения: -- положительное целое число. +- Положительное целое число. - 0 — асинхронные вставки отключены. Значение по умолчанию: `16`. @@ -3675,8 +3675,8 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Возможные значения: -- 0 — отключено. -- 1 — включено. +- 0 — клиент возвращает `OK` даже если вставка данных еще не завершена. +- 1 — клиент возвращает `OK` только после завершения вставки данных. Значение по умолчанию: `1`. From 901e5170a162b5ee10616e77a58e8dc200d9a0f2 Mon Sep 17 00:00:00 2001 From: gyuton Date: Tue, 26 Oct 2021 16:42:19 +0300 Subject: [PATCH 105/173] small fixes --- docs/en/operations/settings/settings.md | 8 ++++---- docs/en/sql-reference/statements/insert-into.md | 4 ++-- docs/ru/operations/settings/settings.md | 8 ++++---- docs/ru/sql-reference/statements/insert-into.md | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 34053ae247a..2287db1b6c0 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3821,7 +3821,7 @@ Enables or disables asynchronous inserts for multiple concurrent connections. Th If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables. -The data is inserted either after the [async_insert_max_data_size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout](#async-insert-busy-timeout) milliseconds since the first `INSERT` query. If the [async_insert_stale_timeout](#async-insert-stale-timeout) is set to a non-zero value, the data is inserted after `async_insert_stale_timeout` milliseconds since the last query. +The data is inserted either after the [async_insert_max_data_size](#async-insert-max-data-size) is exceeded or after [async_insert_busy_timeout_ms](#async-insert-busy-timeout-ms) milliseconds since the first `INSERT` query. If the [async_insert_stale_timeout_ms](#async-insert-stale-timeout-ms) is set to a non-zero value, the data is inserted after `async_insert_stale_timeout_ms` milliseconds since the last query. If [wait_for_async_insert](#wait-for-async-insert) is enabled, every client will wait for the data to be processed and flushed to the table. Otherwise, the query would be processed almost instantly, even if the data is not inserted. @@ -3876,7 +3876,7 @@ Possible values: Default value: `1000000`. -## async_insert_busy_timeout {#async-insert-busy-timeout} +## async_insert_busy_timeout_ms {#async-insert-busy-timeout-ms} The maximum timeout in milliseconds since the first `INSERT` query before inserting collected data. @@ -3887,9 +3887,9 @@ Possible values: Default value: `200`. -## async_insert_stale_timeout {#async-insert-stale-timeout} +## async_insert_stale_timeout_ms {#async-insert-stale-timeout-ms} -The maximum timeout in milliseconds since the last `INSERT` query before dumping collected data. If enabled, the settings prolongs the [async_insert_busy_timeout](#async-insert-busy-timeout) with every `INSERT` query as long as [async_insert_max_data_size](#async-insert-max-data-size) is not exceeded. +The maximum timeout in milliseconds since the last `INSERT` query before dumping collected data. If enabled, the settings prolongs the [async_insert_busy_timeout_ms](#async-insert-busy-timeout-ms) with every `INSERT` query as long as [async_insert_max_data_size](#async-insert-max-data-size) is not exceeded. Possible values: diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index ad5ab705937..0a37a5f41ae 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -128,5 +128,5 @@ It's also possible to asynchronously insert data in small but frequent inserts f - [wait_for_async_insert](../../operations/settings/settings.md#wait-for-async-insert) - [wait_for_async_insert_timeout](../../operations/settings/settings.md#wait-for-async-insert-timeout) - [async_insert_max_data_size](../../operations/settings/settings.md#async-insert-max-data-size) -- [async_insert_busy_timeout](../../operations/settings/settings.md#async-insert-busy-timeout) -- [async_insert_stale_timeout](../../operations/settings/settings.md#async-insert-stale-timeout) +- [async_insert_busy_timeout_ms](../../operations/settings/settings.md#async-insert-busy-timeout-ms) +- [async_insert_stale_timeout_ms](../../operations/settings/settings.md#async-insert-stale-timeout-ms) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 826bd6db77d..ffe52d365d3 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3647,7 +3647,7 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Если включено, данные собираются в пачки перед вставкой в таблицу. Это позволяет производить мелкие и частые вставки в ClickHouse (до 15000 запросов в секунду) без промежуточных таблиц. -Вставка данных происходит либо как только объем вставляемых данных превышает [async_insert_max_data_size](#async-insert-max-data-size), либо через [async_insert_busy_timeout](#async-insert-busy-timeout) миллисекунд после первого запроса `INSERT`. Если в [async_insert_stale_timeout](#async-insert-stale-timeout) задано ненулевое значение, то данные вставляются через `async_insert_stale_timeout` миллисекунд после последнего запроса. +Вставка данных происходит либо как только объем вставляемых данных превышает [async_insert_max_data_size](#async-insert-max-data-size), либо через [async_insert_busy_timeout_ms](#async-insert-busy-timeout-ms) миллисекунд после первого запроса `INSERT`. Если в [async_insert_stale_timeout_ms](#async-insert-stale-timeout-ms) задано ненулевое значение, то данные вставляются через `async_insert_stale_timeout_ms` миллисекунд после последнего запроса. Если включен параметр [wait_for_async_insert](#wait-for-async-insert), каждый клиент ждет, пока данные будут сброшены в таблицу. Иначе запрос будет обработан почти моментально, даже если данные еще не вставлены. @@ -3702,7 +3702,7 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Значение по умолчанию: `1000000`. -## async_insert_busy_timeout {#async-insert-busy-timeout} +## async_insert_busy_timeout_ms {#async-insert-busy-timeout-ms} Максимальное время ожидания в миллисекундах после первого запроса `INSERT` и перед вставкой данных. @@ -3713,9 +3713,9 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Значение по умолчанию: `200`. -## async_insert_stale_timeout {#async-insert-stale-timeout} +## async_insert_stale_timeout_ms {#async-insert-stale-timeout-ms} -Максимальное время ожидания в миллисекундах после последнего запроса `INSERT` и перед вставкой данных. Если установлено ненулевое значение, [async_insert_busy_timeout](#async-insert-busy-timeout) будет продлеваться с каждым запросом `INSERT`, пока не будет превышен [async_insert_max_data_size](#async-insert-max-data-size). +Максимальное время ожидания в миллисекундах после последнего запроса `INSERT` и перед вставкой данных. Если установлено ненулевое значение, [async_insert_busy_timeout_ms](#async-insert-busy-timeout-ms) будет продлеваться с каждым запросом `INSERT`, пока не будет превышен [async_insert_max_data_size](#async-insert-max-data-size). Возможные значения: diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index d7732cc58e1..7fc801b3b88 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -130,5 +130,5 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - [wait_for_async_insert](../../operations/settings/settings.md#wait-for-async-insert) - [wait_for_async_insert_timeout](../../operations/settings/settings.md#wait-for-async-insert-timeout) - [async_insert_max_data_size](../../operations/settings/settings.md#async-insert-max-data-size) -- [async_insert_busy_timeout](../../operations/settings/settings.md#async-insert-busy-timeout) -- [async_insert_stale_timeout](../../operations/settings/settings.md#async-insert-stale-timeout) +- [async_insert_busy_timeout_ms](../../operations/settings/settings.md#async-insert-busy-timeout-ms) +- [async_insert_stale_timeout_ms](../../operations/settings/settings.md#async-insert-stale-timeout-ms) From 6be1e9834249a32ffed02f72ccc020a2ab14e831 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 26 Oct 2021 17:06:48 +0300 Subject: [PATCH 106/173] fix --- src/Storages/MergeTree/MergeTreeData.cpp | 5 ++++- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 15 +++++++++++---- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 57729ef2eda..8dbe79e3cf7 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2692,7 +2692,10 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c void MergeTreeData::forgetPartAndMoveToDetached(const MergeTreeData::DataPartPtr & part_to_detach, const String & prefix, bool restore_covered) { - LOG_INFO(log, "Renaming {} to {}{} and forgetting it.", part_to_detach->relative_path, prefix, part_to_detach->name); + if (prefix.empty()) + LOG_INFO(log, "Renaming {} to {} and forgetting it.", part_to_detach->relative_path, part_to_detach->name); + else + LOG_INFO(log, "Renaming {} to {}_{} and forgetting it.", part_to_detach->relative_path, prefix, part_to_detach->name); auto lock = lockParts(); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 985d54e8aee..2121a9dc09e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -187,13 +187,20 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) void ReplicatedMergeTreeQueue::createLogEntriesToFetchBrokenParts() { - std::lock_guard lock(state_mutex); - if (broken_parts_to_enqueue_fetches_on_loading.empty()) - return; + Strings broken_parts; + { + std::lock_guard lock(state_mutex); + broken_parts = broken_parts_to_enqueue_fetches_on_loading; + } - for (const auto & broken_part_name : broken_parts_to_enqueue_fetches_on_loading) + /// It will lock state_mutex + for (const auto & broken_part_name : broken_parts) storage.removePartAndEnqueueFetch(broken_part_name); + std::lock_guard lock(state_mutex); + /// broken_parts_to_enqueue_fetches_on_loading can be assigned only once on table startup, + /// so actually no race conditions are possible + assert(broken_parts == broken_parts_to_enqueue_fetches_on_loading); broken_parts_to_enqueue_fetches_on_loading.clear(); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f7bccb7e823..053020f7007 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3352,7 +3352,11 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n fs::path(replica_path) / "queue/queue-", log_entry->toString(), zkutil::CreateMode::PersistentSequential)); - zookeeper->multi(ops); + auto results = zookeeper->multi(ops); + + String path_created = dynamic_cast(*results.back()).path_created; + log_entry->znode_name = path_created.substr(path_created.find_last_of('/') + 1); + queue.insert(zookeeper, log_entry); } From d4d97d7b9a1f37ee59729080edfbfde55fb0a57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=8C=E6=B6=9B?= Date: Tue, 28 Sep 2021 16:24:06 +0800 Subject: [PATCH 107/173] add new function mapContainsKeyLike fixed error because of inconsistent offsets --- src/Functions/MatchImpl.h | 5 + src/Functions/like.cpp | 9 -- src/Functions/map.cpp | 122 ++++++++++++++++++ ...2030_function_mapContainsKeyLike.reference | 6 + .../02030_function_mapContainsKeyLike.sql | 12 ++ 5 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/02030_function_mapContainsKeyLike.reference create mode 100644 tests/queries/0_stateless/02030_function_mapContainsKeyLike.sql diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 7dc0712023f..4849fe6e12d 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -419,4 +419,9 @@ struct MatchImpl } }; +struct NameLike +{ + static constexpr auto name = "like"; +}; + } diff --git a/src/Functions/like.cpp b/src/Functions/like.cpp index 1ac9a8d7dab..5d2d5de1be2 100644 --- a/src/Functions/like.cpp +++ b/src/Functions/like.cpp @@ -5,19 +5,10 @@ namespace DB { -namespace -{ - -struct NameLike -{ - static constexpr auto name = "like"; -}; using LikeImpl = MatchImpl; using FunctionLike = FunctionsStringSearch; -} - void registerFunctionLike(FunctionFactory & factory) { factory.registerFunction(); diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 5517dced3e0..f84312ba6dc 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include #include "array/arrayIndex.h" +#include "Functions/MatchImpl.h" +#include "Functions/FunctionsStringSearch.h" namespace DB @@ -28,6 +31,9 @@ namespace ErrorCodes namespace { +using LikeImpl = MatchImpl; +using FunctionLike = FunctionsStringSearch; + // map(x, y, ...) is a function that allows you to make key-value pair class FunctionMap : public IFunction { @@ -274,6 +280,121 @@ public: } }; +class FunctionMapContainsKeyLike : public IFunction +{ +public: + static constexpr auto name = "mapContainsKeyLike"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + String getName() const override { return name; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*info*/) const override { return true; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + bool is_const = isColumnConst(*arguments[0].column); + const ColumnMap * col_map = is_const ? checkAndGetColumnConstData(arguments[0].column.get()) + : checkAndGetColumn(arguments[0].column.get()); + const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); + if (!col_map || !map_type) + throw Exception{"First argument for function " + getName() + " must be a map", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + auto col_res = ColumnVector::create(); + typename ColumnVector::Container & vec_res = col_res->getData(); + + if (input_rows_count == 0) + return col_res; + + vec_res.resize(input_rows_count); + + const auto & column_array = typeid_cast(col_map->getNestedColumn()); + const auto & column_tuple = typeid_cast(column_array.getData()); + + const ColumnString * column_string = checkAndGetColumn(column_tuple.getColumn(0)); + const ColumnFixedString * column_fixed_string = checkAndGetColumn(column_tuple.getColumn(0)); + + FunctionLike func_like; + + for (size_t row = 0; row < input_rows_count; row++) + { + size_t element_start_row = row != 0 ? column_array.getOffsets()[row-1] : 0; + size_t elem_size = column_array.getOffsets()[row]- element_start_row; + + ColumnPtr sub_map_column; + DataTypePtr data_type; + + //The keys of one row map will be processed as a single ColumnString + if (column_string) + { + sub_map_column = column_string->cut(element_start_row, elem_size); + data_type = std::make_shared(); + } + else + { + sub_map_column = column_fixed_string->cut(element_start_row, elem_size); + data_type = std::make_shared(checkAndGetColumn(sub_map_column.get())->getN()); + } + + size_t col_key_size = sub_map_column->size(); + auto column = is_const? ColumnConst::create(std::move(sub_map_column), std::move(col_key_size)) : std::move(sub_map_column); + + ColumnsWithTypeAndName new_arguments = + { + { + column, + data_type, + "" + }, + arguments[1] + }; + + auto res = func_like.executeImpl(new_arguments, result_type, input_rows_count); + + const auto & container = checkAndGetColumn(res.get())->getData(); + + bool exist = 0; + for (auto iter = container.begin(); iter != container.end(); iter++) + { + if (*iter == 1) + { + exist = 1; + break; + } + } + + vec_res[row] = exist; + } + + return col_res; + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 2) + throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " + + toString(arguments.size()) + ", should be 2", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeMap * map_type = checkAndGetDataType(arguments[0].type.get()); + const DataTypeString * pattern_type = checkAndGetDataType(arguments[1].type.get()); + + if (!map_type) + throw Exception{"First argument for function " + getName() + " must be a Map", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + if (!pattern_type) + throw Exception{"Second argument for function " + getName() + " must be String", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + if (!isStringOrFixedString(map_type->getKeyType())) + throw Exception{"Key type of map for function " + getName() + " must be `String` or `FixedString`", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + return std::make_shared(); + } + + size_t getNumberOfArguments() const override { return 2; } + + bool useDefaultImplementationForConstants() const override { return true; } +}; + } void registerFunctionsMap(FunctionFactory & factory) @@ -282,6 +403,7 @@ void registerFunctionsMap(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/tests/queries/0_stateless/02030_function_mapContainsKeyLike.reference b/tests/queries/0_stateless/02030_function_mapContainsKeyLike.reference new file mode 100644 index 00000000000..eb8bc83384e --- /dev/null +++ b/tests/queries/0_stateless/02030_function_mapContainsKeyLike.reference @@ -0,0 +1,6 @@ +1 {'1-K1':'1-V1','1-K2':'1-V2'} +1 {'1-K1':'1-V1','1-K2':'1-V2'} +2 {'2-K1':'2-V1','2-K2':'2-V2'} +4 {'4-K1':'4-V1','4-K2':'4-V2'} +5 {'5-K1':'5-V1','5-K2':'5-V2'} +6 {'6-K1':'6-V1','6-K2':'6-V2'} diff --git a/tests/queries/0_stateless/02030_function_mapContainsKeyLike.sql b/tests/queries/0_stateless/02030_function_mapContainsKeyLike.sql new file mode 100644 index 00000000000..7d9722b4c90 --- /dev/null +++ b/tests/queries/0_stateless/02030_function_mapContainsKeyLike.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS map_containsKeyLike_test; + +CREATE TABLE map_containsKeyLike_test (id UInt32, map Map(String, String)) Engine=MergeTree() ORDER BY id settings index_granularity=2; + +INSERT INTO map_containsKeyLike_test VALUES (1, {'1-K1':'1-V1','1-K2':'1-V2'}),(2,{'2-K1':'2-V1','2-K2':'2-V2'}); +INSERT INTO map_containsKeyLike_test VALUES (3, {'3-K1':'3-V1','3-K2':'3-V2'}),(4, {'4-K1':'4-V1','4-K2':'4-V2'}); +INSERT INTO map_containsKeyLike_test VALUES (5, {'5-K1':'5-V1','5-K2':'5-V2'}),(6, {'6-K1':'6-V1','6-K2':'6-V2'}); + +SELECT id, map FROM map_containsKeyLike_test WHERE mapContainsKeyLike(map, '1-%') = 1; +SELECT id, map FROM map_containsKeyLike_test WHERE mapContainsKeyLike(map, '3-%') = 0 order by id; + +DROP TABLE map_containsKeyLike_test; From 1caff8a81c713d244f9d51bd7984c822ca036af3 Mon Sep 17 00:00:00 2001 From: jasperzhu Date: Tue, 26 Oct 2021 22:12:21 +0530 Subject: [PATCH 108/173] fixed compile error --- src/Columns/ColumnsCommon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index 2be9b18de4e..3b27ca9bd71 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -41,7 +41,7 @@ UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64) const UInt8 * pos = bytes64; const UInt8 * end = pos + 64; for (; pos < end; ++pos) - res |= (*pos == 0)<<(pos-bytes64)); + res |= ((*pos == 0)<<(pos-bytes64)); #endif return ~res; } From bc0a9c69848dd24b66b42a059c708827bb6dd58a Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Tue, 26 Oct 2021 20:09:13 +0300 Subject: [PATCH 109/173] Update index.md --- docs/en/sql-reference/operators/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index d23c81e2aca..61a8341db14 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -73,11 +73,11 @@ ClickHouse transforms operators to their corresponding functions at the query pa `a = ANY (subquery)` – The `in(a, subquery)` function. -`a != ANY (subquery)` – The `notIn(a, subquery)` function. +`a != ANY (subquery)` – The same as `a NOT IN (SELECT singleValueOrNull(*) FROM subquery)`. -`a = ALL (subquery)` – The same as `a IN (SELECT singleValueOrNull(*) FROM subquery)`. +`a = ALL (subquery)` – The same as `a IN (SELECT singleValueOrNull(*) FROM subquery)`. -`a != ALL (subquery)` – The same as `a NOT IN (subquery)`. +`a != ALL (subquery)` – The `notIn(a, subquery)` function. **Examples** From da7d491125495f8d91a3bc159666b34b488f2169 Mon Sep 17 00:00:00 2001 From: Tatiana Kirillova Date: Tue, 26 Oct 2021 20:10:30 +0300 Subject: [PATCH 110/173] Update index.md --- docs/ru/sql-reference/operators/index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 19119f641ef..2d67eb1348b 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -72,13 +72,13 @@ toc_title: "Операторы" `a GLOBAL NOT IN ...` - функция `globalNotIn(a, b)` -`a = ANY (subquery)` – функция `in(a, subquery)` +`a = ANY (subquery)` – функция `in(a, subquery)`. -`a != ANY (subquery)` – функция `notIn(a, subquery)` +`a != ANY (subquery)` – равнозначно `a NOT IN (SELECT singleValueOrNull(*) FROM subquery)`. -`a = ALL (subquery)` – равнозначно `a IN (SELECT singleValueOrNull(*) FROM subquery)` +`a = ALL (subquery)` – равнозначно `a IN (SELECT singleValueOrNull(*) FROM subquery)`. -`a != ALL (subquery)` – равнозначно `a NOT IN (subquery)` +`a != ALL (subquery)` – функция `notIn(a, subquery)`. **Примеры** From 403ff98baa41adff67127871fa65c3a99700e15d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 Oct 2021 20:15:29 +0300 Subject: [PATCH 111/173] Revert "Improve usability of `remote_url_allow_hosts`" --- src/Common/RemoteHostFilter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/RemoteHostFilter.cpp b/src/Common/RemoteHostFilter.cpp index ba7f163fd16..73c84364f3c 100644 --- a/src/Common/RemoteHostFilter.cpp +++ b/src/Common/RemoteHostFilter.cpp @@ -18,14 +18,14 @@ void RemoteHostFilter::checkURL(const Poco::URI & uri) const { if (!checkForDirectEntry(uri.getHost()) && !checkForDirectEntry(uri.getHost() + ":" + toString(uri.getPort()))) - throw Exception("URL \"" + uri.toString() + "\" is not allowed in configuration file, see ", ErrorCodes::UNACCEPTABLE_URL); + throw Exception("URL \"" + uri.toString() + "\" is not allowed in config.xml", ErrorCodes::UNACCEPTABLE_URL); } void RemoteHostFilter::checkHostAndPort(const std::string & host, const std::string & port) const { if (!checkForDirectEntry(host) && !checkForDirectEntry(host + ":" + port)) - throw Exception("URL \"" + host + ":" + port + "\" is not allowed in configuration file, see ", ErrorCodes::UNACCEPTABLE_URL); + throw Exception("URL \"" + host + ":" + port + "\" is not allowed in config.xml", ErrorCodes::UNACCEPTABLE_URL); } void RemoteHostFilter::setValuesFromConfig(const Poco::Util::AbstractConfiguration & config) From aae30a9e37d71fa84d828cf70352cad7f7f4507c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 26 Oct 2021 20:30:34 +0300 Subject: [PATCH 112/173] Fix fuzzer build (#30344) --- CMakeLists.txt | 2 -- contrib/protobuf-cmake/CMakeLists.txt | 10 ++++++++++ docker/packager/other/fuzzer.sh | 12 ++++++------ src/Interpreters/fuzzers/CMakeLists.txt | 1 - src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1790ddc0451..ede70b09d94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,8 +149,6 @@ if (ENABLE_FUZZING) set (ENABLE_JEMALLOC 0) set (ENABLE_CHECK_HEAVY_BUILDS 1) set (GLIBC_COMPATIBILITY OFF) - set (ENABLE_PROTOBUF ON) - set (USE_INTERNAL_PROTOBUF_LIBRARY ON) endif() # Global libraries diff --git a/contrib/protobuf-cmake/CMakeLists.txt b/contrib/protobuf-cmake/CMakeLists.txt index 0ceb72cfbd6..37215b91dbb 100644 --- a/contrib/protobuf-cmake/CMakeLists.txt +++ b/contrib/protobuf-cmake/CMakeLists.txt @@ -17,6 +17,16 @@ endif () add_subdirectory("${protobuf_SOURCE_DIR}/cmake" "${protobuf_BINARY_DIR}") +if (ENABLE_FUZZING) + # `protoc` will be built with sanitizer and it could fail during ClickHouse build + # It easily reproduces in oss-fuzz building pipeline + # To avoid this we can try to build `protoc` without any sanitizer with option `-fno-sanitize=all`, but + # it this case we will face with linker errors, because libcxx still will be built with sanitizer + # So, we can simply suppress all of these failures with a combination this flag and an environment variable + # export MSAN_OPTIONS=exit_code=0 + target_compile_options(protoc PRIVATE "-fsanitize-recover=all") +endif() + # We don't want to stop compilation on warnings in protobuf's headers. # The following line overrides the value assigned by the command target_include_directories() in libprotobuf.cmake set_property(TARGET libprotobuf PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${protobuf_SOURCE_DIR}/src") diff --git a/docker/packager/other/fuzzer.sh b/docker/packager/other/fuzzer.sh index 5eec51f9625..baaf3c4bedf 100755 --- a/docker/packager/other/fuzzer.sh +++ b/docker/packager/other/fuzzer.sh @@ -12,19 +12,19 @@ printenv rm -f CMakeCache.txt read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}" # Hope, that the most part of files will be in cache, so we just link new executables -cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_CLICKHOUSE_ODBC_BRIDGE=OFF \ - -DENABLE_LIBRARIES=0 -DENABLE_SSL=1 -DUSE_INTERNAL_SSL_LIBRARY=1 -DUSE_UNWIND=ON -DENABLE_EMBEDDED_COMPILER=0 \ - -DENABLE_EXAMPLES=0 -DENABLE_UTILS=0 -DENABLE_THINLTO=0 "-DSANITIZE=$SANITIZER" \ - -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_TCMALLOC=0 -DENABLE_JEMALLOC=0 \ - -DENABLE_CHECK_HEAVY_BUILDS=1 -DGLIBC_COMPATIBILITY=OFF "${CMAKE_FLAGS[@]}" .. +# Please, add or change flags directly in cmake +cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ + -DSANITIZE="$SANITIZER" -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_PROTOBUF=1 -DUSE_INTERNAL_PROTOBUF_LIBRARY=1 "${CMAKE_FLAGS[@]}" .. FUZZER_TARGETS=$(find ../src -name '*_fuzzer.cpp' -execdir basename {} .cpp ';' | tr '\n' ' ') +NUM_JOBS=$(($(nproc || grep -c ^processor /proc/cpuinfo))) + mkdir -p /output/fuzzers for FUZZER_TARGET in $FUZZER_TARGETS do # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. - ninja $NINJA_FLAGS $FUZZER_TARGET + ninja $NINJA_FLAGS $FUZZER_TARGET -j $NUM_JOBS # Find this binary in build directory and strip it FUZZER_PATH=$(find ./src -name "$FUZZER_TARGET") strip --strip-unneeded "$FUZZER_PATH" diff --git a/src/Interpreters/fuzzers/CMakeLists.txt b/src/Interpreters/fuzzers/CMakeLists.txt index dfe892b5249..f8d4cf224db 100644 --- a/src/Interpreters/fuzzers/CMakeLists.txt +++ b/src/Interpreters/fuzzers/CMakeLists.txt @@ -7,4 +7,3 @@ target_link_libraries(execute_query_fuzzer PRIVATE clickhouse_dictionaries clickhouse_dictionaries_embedded ${LIB_FUZZING_ENGINE}) - diff --git a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt index 6f6b0d07661..d5e6c944852 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt +++ b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt @@ -45,4 +45,4 @@ set_source_files_properties("${PROTO_SRCS}" "out.cpp" PROPERTIES COMPILE_FLAGS " target_include_directories(codegen_select_fuzzer BEFORE PRIVATE "${Protobuf_INCLUDE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") target_include_directories(codegen_select_fuzzer BEFORE PRIVATE "${LibProtobufMutator_SOURCE_DIR}") target_include_directories(codegen_select_fuzzer BEFORE PRIVATE "${LibProtobufMutator_SOURCE_DIR}/src") -target_link_libraries(codegen_select_fuzzer PRIVATE protobuf-mutator dbms ${LIB_FUZZING_ENGINE}) +target_link_libraries(codegen_select_fuzzer PRIVATE protobuf-mutator ${Protobuf_LIBRARY} ${Protobuf_PROTOC_LIBRARY} dbms ${LIB_FUZZING_ENGINE}) From fb31410926fd3d3097b983e928468274f4f1d2d6 Mon Sep 17 00:00:00 2001 From: jasperzhu Date: Wed, 27 Oct 2021 00:05:47 +0530 Subject: [PATCH 113/173] change to inline for bitmask generator --- src/CMakeLists.txt | 3 ++- src/Columns/ColumnsCommon.cpp | 38 +------------------------------- src/Columns/ColumnsCommon.h | 41 ++++++++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e1f25c21d9..5fc15b1609d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -290,12 +290,13 @@ target_link_libraries (clickhouse_common_io ) # Use X86 AVX2/AVX512 instructions to accelerate filter operations +# Add flag:-force-vector-width=16 due to clang13 not support auto vectorize with AVX512, maybe remove it in future. set_source_files_properties( Columns/ColumnFixedString.cpp Columns/ColumnsCommon.cpp Columns/ColumnVector.cpp Columns/ColumnDecimal.cpp - PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}") + PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS} -mprefer-vector-width=256") if(RE2_LIBRARY) target_link_libraries(clickhouse_common_io PUBLIC ${RE2_LIBRARY}) diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index 3b27ca9bd71..315376cadec 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -1,6 +1,4 @@ -#ifdef __SSE2__ - #include -#endif + #include #include @@ -12,40 +10,6 @@ namespace DB { -/// Transform 64-byte mask to 64-bit mask -UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64) -{ -#if defined(__AVX512F__) && defined(__AVX512BW__) - static const __m512i zero64 = _mm512_setzero_epi32(); - UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_EQ); -#elif defined(__AVX__) && defined(__AVX2__) - static const __m256i zero32 = _mm256_setzero_si256(); - UInt64 res = - (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8( - _mm256_loadu_si256(reinterpret_cast(bytes64)), zero32))) & 0xffffffff) - | (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8( - _mm256_loadu_si256(reinterpret_cast(bytes64+32)), zero32))) << 32); -#elif defined(__SSE2__) && defined(__POPCNT__) - static const __m128i zero16 = _mm_setzero_si128(); - UInt64 res = - (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) & 0xffff) - | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) & 0xffff0000) - | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) & 0xffff00000000) - | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( - _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48) & 0xffff000000000000); -#else - UInt64 res = 0; - const UInt8 * pos = bytes64; - const UInt8 * end = pos + 64; - for (; pos < end; ++pos) - res |= ((*pos == 0)<<(pos-bytes64)); -#endif - return ~res; -} - size_t countBytesInFilter(const UInt8 * filt, size_t sz) { size_t count = 0; diff --git a/src/Columns/ColumnsCommon.h b/src/Columns/ColumnsCommon.h index baf26deb00c..c2feb0b31c9 100644 --- a/src/Columns/ColumnsCommon.h +++ b/src/Columns/ColumnsCommon.h @@ -2,8 +2,12 @@ #include #include - - +#ifdef __SSE2__ +#include +#endif +#if defined(__AVX512F__) || defined(__AVX512BW__) || defined(__AVX__) || defined(__AVX2__) +#include +#endif /// Common helper methods for implementation of different columns. namespace DB @@ -16,7 +20,38 @@ namespace ErrorCodes } /// Transform 64-byte mask to 64-bit mask -UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64); +inline UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64) +{ +#if defined(__AVX512F__) && defined(__AVX512BW__) + static const __m512i zero64 = _mm512_setzero_epi32(); + UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_GT); +#elif defined(__AVX__) && defined(__AVX2__) + static const __m256i zero32 = _mm256_setzero_si256(); + UInt64 res = + (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + _mm256_loadu_si256(reinterpret_cast(bytes64)), zero32))) & 0xffffffff) + | (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + _mm256_loadu_si256(reinterpret_cast(bytes64+32)), zero32))) << 32); +#elif defined(__SSE2__) && defined(__POPCNT__) + static const __m128i zero16 = _mm_setzero_si128(); + UInt64 res = + static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) + | (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48); +#else + UInt64 res = 0; + const UInt8 * pos = bytes64; + const UInt8 * end = pos + 64; + for (; pos < end; ++pos) + res |= ((*pos != 0)<<(pos-bytes64)); +#endif + return res; +} /// Counts how many bytes of `filt` are greater than zero. size_t countBytesInFilter(const UInt8 * filt, size_t sz); size_t countBytesInFilter(const IColumn::Filter & filt); From 874f7a505263aabf3f5bb949d707f065fa2526ef Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 22 Oct 2021 00:12:30 +0300 Subject: [PATCH 114/173] Add a test with optimized columns from PREWHERE --- .../02021_prewhere_column_optimization.reference | 7 +++++++ .../0_stateless/02021_prewhere_column_optimization.sql | 10 ++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/02021_prewhere_column_optimization.reference create mode 100644 tests/queries/0_stateless/02021_prewhere_column_optimization.sql diff --git a/tests/queries/0_stateless/02021_prewhere_column_optimization.reference b/tests/queries/0_stateless/02021_prewhere_column_optimization.reference new file mode 100644 index 00000000000..7f97e6efd69 --- /dev/null +++ b/tests/queries/0_stateless/02021_prewhere_column_optimization.reference @@ -0,0 +1,7 @@ +-- { echoOn } +select * from data_02021 prewhere 1 or ignore(key); +1 +select * from data_02021 prewhere 1 or ignore(key) where key = 1; +1 +select * from data_02021 prewhere 0 or ignore(key); +select * from data_02021 prewhere 0 or ignore(key) where key = 1; diff --git a/tests/queries/0_stateless/02021_prewhere_column_optimization.sql b/tests/queries/0_stateless/02021_prewhere_column_optimization.sql new file mode 100644 index 00000000000..4fe8b912c3f --- /dev/null +++ b/tests/queries/0_stateless/02021_prewhere_column_optimization.sql @@ -0,0 +1,10 @@ +drop table if exists data_02021; +create table data_02021 (key Int) engine=MergeTree() order by key; +insert into data_02021 values (1); +-- { echoOn } +select * from data_02021 prewhere 1 or ignore(key); +select * from data_02021 prewhere 1 or ignore(key) where key = 1; +select * from data_02021 prewhere 0 or ignore(key); +select * from data_02021 prewhere 0 or ignore(key) where key = 1; +-- { echoOff } +drop table data_02021; From 248b689067e3c90d68673ae86678389992ae4316 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 22 Oct 2021 00:15:59 +0300 Subject: [PATCH 115/173] ActionsDag: add allow_remove_inputs for other variants of removeUnusedActions() --- src/Interpreters/ActionsDAG.cpp | 8 ++++---- src/Interpreters/ActionsDAG.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index ea9ee2ce542..56266d50e9d 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -325,7 +325,7 @@ std::string ActionsDAG::dumpNames() const return out.str(); } -void ActionsDAG::removeUnusedActions(const NameSet & required_names) +void ActionsDAG::removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs) { NodeRawConstPtrs required_nodes; required_nodes.reserve(required_names.size()); @@ -349,10 +349,10 @@ void ActionsDAG::removeUnusedActions(const NameSet & required_names) } index.swap(required_nodes); - removeUnusedActions(); + removeUnusedActions(allow_remove_inputs); } -void ActionsDAG::removeUnusedActions(const Names & required_names) +void ActionsDAG::removeUnusedActions(const Names & required_names, bool allow_remove_inputs) { NodeRawConstPtrs required_nodes; required_nodes.reserve(required_names.size()); @@ -372,7 +372,7 @@ void ActionsDAG::removeUnusedActions(const Names & required_names) } index.swap(required_nodes); - removeUnusedActions(); + removeUnusedActions(allow_remove_inputs); } void ActionsDAG::removeUnusedActions(bool allow_remove_inputs) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 8111bcb123c..9baab1231d4 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -162,8 +162,8 @@ public: bool isInputProjected() const { return project_input; } bool isOutputProjected() const { return projected_output; } - void removeUnusedActions(const Names & required_names); - void removeUnusedActions(const NameSet & required_names); + void removeUnusedActions(const Names & required_names, bool allow_remove_inputs = true); + void removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs = true); NameSet foldActionsByProjection( const NameSet & required_columns, From dd7bffcaf96a5d143d77d65e3d0510bf53c0bfd0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 26 Oct 2021 22:56:12 +0300 Subject: [PATCH 116/173] ActionsDAG: add allow_constant_folding flag for removeUnusedActions() --- src/Interpreters/ActionsDAG.cpp | 14 +++++++------- src/Interpreters/ActionsDAG.h | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 56266d50e9d..d0b360dda82 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -325,7 +325,7 @@ std::string ActionsDAG::dumpNames() const return out.str(); } -void ActionsDAG::removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs) +void ActionsDAG::removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs, bool allow_constant_folding) { NodeRawConstPtrs required_nodes; required_nodes.reserve(required_names.size()); @@ -349,10 +349,10 @@ void ActionsDAG::removeUnusedActions(const NameSet & required_names, bool allow_ } index.swap(required_nodes); - removeUnusedActions(allow_remove_inputs); + removeUnusedActions(allow_remove_inputs, allow_constant_folding); } -void ActionsDAG::removeUnusedActions(const Names & required_names, bool allow_remove_inputs) +void ActionsDAG::removeUnusedActions(const Names & required_names, bool allow_remove_inputs, bool allow_constant_folding) { NodeRawConstPtrs required_nodes; required_nodes.reserve(required_names.size()); @@ -372,10 +372,10 @@ void ActionsDAG::removeUnusedActions(const Names & required_names, bool allow_re } index.swap(required_nodes); - removeUnusedActions(allow_remove_inputs); + removeUnusedActions(allow_remove_inputs, allow_constant_folding); } -void ActionsDAG::removeUnusedActions(bool allow_remove_inputs) +void ActionsDAG::removeUnusedActions(bool allow_remove_inputs, bool allow_constant_folding) { std::unordered_set visited_nodes; std::stack stack; @@ -406,9 +406,9 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs) auto * node = stack.top(); stack.pop(); - if (!node->children.empty() && node->column && isColumnConst(*node->column)) + /// Constant folding. + if (allow_constant_folding && !node->children.empty() && node->column && isColumnConst(*node->column)) { - /// Constant folding. node->type = ActionsDAG::ActionType::COLUMN; for (const auto & child : node->children) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 9baab1231d4..95189416cc3 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -162,8 +162,8 @@ public: bool isInputProjected() const { return project_input; } bool isOutputProjected() const { return projected_output; } - void removeUnusedActions(const Names & required_names, bool allow_remove_inputs = true); - void removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs = true); + void removeUnusedActions(const Names & required_names, bool allow_remove_inputs = true, bool allow_constant_folding = true); + void removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs = true, bool allow_constant_folding = true); NameSet foldActionsByProjection( const NameSet & required_columns, @@ -271,7 +271,7 @@ public: private: Node & addNode(Node node); - void removeUnusedActions(bool allow_remove_inputs = true); + void removeUnusedActions(bool allow_remove_inputs = true, bool allow_constant_folding = true); #if USE_EMBEDDED_COMPILER void compileFunctions(size_t min_count_to_compile_expression, const std::unordered_set & lazy_executed_nodes = {}); From fbf98bea0ba56440b973fabbfe755aeb13f078ef Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 2 Sep 2021 08:35:30 +0300 Subject: [PATCH 117/173] Fix "Column is not under aggregate function and not in GROUP BY" with PREWHERE In 21.7 the function column type is unknown: (lldb) p tmp_actions_dag->dumpDAG() (std::string) $14 = "0 : INPUT () (no column) Int32 key\n1 : COLUMN () Const(UInt8) UInt8 1\n2 : FUNCTION (0) (no column) UInt8 ignore(key) [ignore]\n3 : FUNCTION (1, 2) (no column) UInt8 or(1, ignore(key)) [or]\nIndex: 0 1 2 3\n" ^^^^^^^^^^^ While in 21.8+ it is const: (lldb) p tmp_actions_dag->dumpDAG() (std::string) $22 = "0 : INPUT () (no column) Int32 key\n1 : COLUMN () Const(UInt8) UInt8 1\n2 : FUNCTION (0) (no column) UInt8 ignore(key) [ignore]\n3 : FUNCTION (1, 2) Const(UInt8) UInt8 or(1, ignore(key)) [or]\nIndex: 0 1 2 3\n" ^^^^^ This is after getRootActions() in SelectQueryExpressionAnalyzer::appendPrewhere() v1: fix for SELECT only, but breaks virtual columns v2: hacky fix, that also touches MergeTree code v3: allow_remove_inputs=false v4: allow_constant_folding=false instead --- src/Interpreters/ExpressionAnalyzer.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 501fc9b4b1d..d40a92972b2 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1045,7 +1045,12 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere( /// Remove unused source_columns from prewhere actions. auto tmp_actions_dag = std::make_shared(sourceColumns()); getRootActions(select_query->prewhere(), only_types, tmp_actions_dag); - tmp_actions_dag->removeUnusedActions(NameSet{prewhere_column_name}); + /// Constants cannot be removed since they can be used in other parts of the query. + /// And if they are not used anywhere, except PREWHERE, they will be removed on the next step. + tmp_actions_dag->removeUnusedActions( + NameSet{prewhere_column_name}, + /* allow_remove_inputs= */ true, + /* allow_constant_folding= */ false); auto required_columns = tmp_actions_dag->getRequiredColumnsNames(); NameSet required_source_columns(required_columns.begin(), required_columns.end()); From b764b6ce216166aa0b571430c40fe0b5d1d17153 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Tue, 26 Oct 2021 15:08:05 -0600 Subject: [PATCH 118/173] Update hardware page colors (#30719) Replaces yellow with a light gray --- website/benchmark/benchmark.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/benchmark/benchmark.js b/website/benchmark/benchmark.js index 8fb2693aa97..bd5ec3c083e 100644 --- a/website/benchmark/benchmark.js +++ b/website/benchmark/benchmark.js @@ -58,7 +58,7 @@ function generate_selectors(elem) { var available = available_systems_for_current_data_size.indexOf(systems[i]) != -1; var button_class = 'btn-outline-dark'; if (system_kinds[i] == 'cloud' || system_kinds[i] == 'vps') { - button_class = 'btn-outline-primary'; + button_class = 'btn-light'; } else if (system_kinds[i] == 'desktop' || system_kinds[i] == 'laptop') { button_class = 'btn-outline-secondary'; }; @@ -424,11 +424,11 @@ function generate_diagram() { } - table_row += '
::filter(const IColumn::Filter & filt, ssize_t result_ while (filt_pos < filt_end_aligned) { - UInt64 mask = bytes64MaskTobits64Mask(filt_pos); + UInt64 mask = Bytes64MaskToBits64Mask(filt_pos); if (0xffffffffffffffff == mask) { diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 5cdc7242208..cc4e7b12a04 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -242,7 +242,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result while (filt_pos < filt_end_aligned) { - UInt64 mask = bytes64MaskTobits64Mask(filt_pos); + uint64_t mask = Bytes64MaskToBits64Mask(filt_pos); if (0xffffffffffffffff == mask) { diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 4a634c35553..b9079857aa4 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -321,7 +321,7 @@ ColumnPtr ColumnVector::filter(const IColumn::Filter & filt, ssize_t result_s while (filt_pos < filt_end_aligned) { - UInt64 mask = bytes64MaskTobits64Mask(filt_pos); + UInt64 mask = Bytes64MaskToBits64Mask(filt_pos); if (0xffffffffffffffff == mask) { diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index f5123b318f7..4632c021896 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -239,7 +239,7 @@ namespace while (filt_pos < filt_end_aligned) { - UInt64 mask = bytes64MaskTobits64Mask(filt_pos); + uint64_t mask = Bytes64MaskToBits64Mask(filt_pos); if (0xffffffffffffffff == mask) { diff --git a/src/Columns/ColumnsCommon.h b/src/Columns/ColumnsCommon.h index f29bc498c3a..d9800054457 100644 --- a/src/Columns/ColumnsCommon.h +++ b/src/Columns/ColumnsCommon.h @@ -16,37 +16,37 @@ namespace ErrorCodes } /// Transform 64-byte mask to 64-bit mask -inline UInt64 bytes64MaskTobits64Mask(const UInt8 * bytes64) +inline UInt64 Bytes64MaskToBits64Mask(const UInt8 * bytes64) { #if defined(__AVX512F__) && defined(__AVX512BW__) static const __m512i zero64 = _mm512_setzero_epi32(); - UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_GT); + UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast(bytes64)), zero64, _MM_CMPINT_EQ); #elif defined(__AVX__) && defined(__AVX2__) static const __m256i zero32 = _mm256_setzero_si256(); UInt64 res = - (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8( _mm256_loadu_si256(reinterpret_cast(bytes64)), zero32))) & 0xffffffff) - | (static_cast(_mm256_movemask_epi8(_mm256_cmpgt_epi8( + | (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8( _mm256_loadu_si256(reinterpret_cast(bytes64+32)), zero32))) << 32); #elif defined(__SSE2__) && defined(__POPCNT__) static const __m128i zero16 = _mm_setzero_si128(); UInt64 res = - (static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( _mm_loadu_si128(reinterpret_cast(bytes64)), zero16))) & 0xffff) - | ((static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( _mm_loadu_si128(reinterpret_cast(bytes64 + 16)), zero16))) << 16) & 0xffff0000) - | ((static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( _mm_loadu_si128(reinterpret_cast(bytes64 + 32)), zero16))) << 32) & 0xffff00000000) - | ((static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8( + | ((static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8( _mm_loadu_si128(reinterpret_cast(bytes64 + 48)), zero16))) << 48) & 0xffff000000000000); #else UInt64 res = 0; const UInt8 * pos = bytes64; const UInt8 * end = pos + 64; for (; pos < end; ++pos) - res |= ((*pos > 0) << (pos - bytes64)); + res |= ((*pos == 0)<<(pos-bytes64)); #endif - return res; + return ~res; } /// Counts how many bytes of `filt` are greater than zero. From 49dd69c6c9c5d36a1d0f6a212e59596f214cb3e7 Mon Sep 17 00:00:00 2001 From: jasperzhu Date: Wed, 27 Oct 2021 05:22:05 +0530 Subject: [PATCH 122/173] adjust header files --- src/Columns/ColumnsCommon.cpp | 4 ---- src/Columns/ColumnsCommon.h | 7 ++++++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Columns/ColumnsCommon.cpp b/src/Columns/ColumnsCommon.cpp index 4632c021896..6d16790e2f8 100644 --- a/src/Columns/ColumnsCommon.cpp +++ b/src/Columns/ColumnsCommon.cpp @@ -1,7 +1,3 @@ -#ifdef __SSE2__ - #include -#endif - #include #include #include diff --git a/src/Columns/ColumnsCommon.h b/src/Columns/ColumnsCommon.h index d9800054457..b84b1de7d67 100644 --- a/src/Columns/ColumnsCommon.h +++ b/src/Columns/ColumnsCommon.h @@ -2,7 +2,12 @@ #include #include - +#ifdef __SSE2__ +#include +#endif +#if defined(__AVX512F__) || defined(__AVX512BW__) || defined(__AVX__) || defined(__AVX2__) +#include +#endif /// Common helper methods for implementation of different columns. From fb6716f4acb902b0947989cde8db613e4f7cce77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=8C=E6=B6=9B?= Date: Wed, 27 Oct 2021 11:56:55 +0800 Subject: [PATCH 123/173] add doc --- .../functions/tuple-map-functions.md | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 6ddac9a0530..46ce350377c 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -350,6 +350,45 @@ Result: │ ['eleven','11'] │ │ ['twelve','6.0'] │ └──────────────────┘ -``` +``` + +## mapContainsKeyLike {#mapContainsKeyLike} + +**Syntax** + +```sql +mapContainsKeyLike(map, pattern) +``` + +**Parameters** + +- `map` — Map. [Map](../../sql-reference/data-types/map.md). +- `pattern` - String pattern to match. + +**Returned value** + +- `1` if `map` contains `key` like specified pattern, `0` if not. + +**Example** + +Query: + +```sql +CREATE TABLE test (a Map(String,String)) ENGINE = Memory; + +INSERT INTO test VALUES ({'abc':'abc','def':'def'}), ({'hij':'hij','klm':'klm'}); + +SELECT mapContainsKeyLike(a, 'a%') FROM test; +``` + +Result: + +```text +┌─mapContainsKeyLike(a, 'a%')─┐ +│ 1 │ +│ 0 │ +└─────────────────────────────┘ +``` + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/tuple-map-functions/) From 4ac120b883d9ca27b080b8bf63ddad96d2d65364 Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 27 Oct 2021 04:13:12 +0000 Subject: [PATCH 124/173] remove some unneeded header files --- src/Storages/FileLog/DirectoryWatcherBase.cpp | 1 - src/Storages/FileLog/FileLogSource.h | 1 - src/Storages/FileLog/ReadBufferFromFileLog.cpp | 1 - src/Storages/FileLog/ReadBufferFromFileLog.h | 2 -- src/Storages/FileLog/StorageFileLog.cpp | 3 --- 5 files changed, 8 deletions(-) diff --git a/src/Storages/FileLog/DirectoryWatcherBase.cpp b/src/Storages/FileLog/DirectoryWatcherBase.cpp index c459079ec06..37a82ad4ee3 100644 --- a/src/Storages/FileLog/DirectoryWatcherBase.cpp +++ b/src/Storages/FileLog/DirectoryWatcherBase.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include diff --git a/src/Storages/FileLog/FileLogSource.h b/src/Storages/FileLog/FileLogSource.h index e1348de5bdf..c9fd1cc5a79 100644 --- a/src/Storages/FileLog/FileLogSource.h +++ b/src/Storages/FileLog/FileLogSource.h @@ -1,7 +1,6 @@ #pragma once #include - #include #include diff --git a/src/Storages/FileLog/ReadBufferFromFileLog.cpp b/src/Storages/FileLog/ReadBufferFromFileLog.cpp index a55df9fe09e..86caac57a53 100644 --- a/src/Storages/FileLog/ReadBufferFromFileLog.cpp +++ b/src/Storages/FileLog/ReadBufferFromFileLog.cpp @@ -6,7 +6,6 @@ #include #include -#include namespace DB { diff --git a/src/Storages/FileLog/ReadBufferFromFileLog.h b/src/Storages/FileLog/ReadBufferFromFileLog.h index 117a858de3b..d581ead951e 100644 --- a/src/Storages/FileLog/ReadBufferFromFileLog.h +++ b/src/Storages/FileLog/ReadBufferFromFileLog.h @@ -1,10 +1,8 @@ #pragma once #include -#include #include #include -#include #include #include diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 463ad77b142..d9b08762d28 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -24,8 +23,6 @@ #include #include #include -#include -#include #include From d0a031014e46a26ef84aeb2ce5f491c39d56e693 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 09:26:23 +0300 Subject: [PATCH 125/173] Fix test_part_uuid::test_part_uuid_wal CI: https://clickhouse-test-reports.s3.yandex.net/30716/ca41267a4d5d7e1da3dd584c98a9143e15f394cf/integration_tests_(asan).html#fail1 --- tests/integration/test_part_uuid/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_part_uuid/test.py b/tests/integration/test_part_uuid/test.py index 1dfeb17b9b8..0353bf9266d 100644 --- a/tests/integration/test_part_uuid/test.py +++ b/tests/integration/test_part_uuid/test.py @@ -93,7 +93,7 @@ def test_part_uuid_wal(started_cluster): uuids = set() for node in [node1, node2]: - node.query("SYSTEM SYNC REPLICA t") + node.query("SYSTEM SYNC REPLICA t_wal") part_initial_uuid = uuid.UUID(node.query("SELECT uuid FROM system.parts WHERE table = 't_wal' AND active ORDER BY name").strip()) assert "InMemory" == node.query("SELECT part_type FROM system.parts WHERE table = 't_wal' AND active ORDER BY name").strip() uuids.add(part_initial_uuid) From 7ce01b328ec0937045e30eb19c490b8c304c4041 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 26 Oct 2021 23:53:24 +0300 Subject: [PATCH 126/173] Update gtest_disk_encrypted for new createReadBufferFromFileBase() interface --- src/Disks/tests/gtest_disk_encrypted.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/tests/gtest_disk_encrypted.cpp b/src/Disks/tests/gtest_disk_encrypted.cpp index 6f49f467734..60cf8d81d92 100644 --- a/src/Disks/tests/gtest_disk_encrypted.cpp +++ b/src/Disks/tests/gtest_disk_encrypted.cpp @@ -65,7 +65,7 @@ protected: static String getBinaryRepresentation(const String & abs_path) { - auto buf = createReadBufferFromFileBase(abs_path, {}, 0); + auto buf = createReadBufferFromFileBase(abs_path, /* settings= */ {}); String str; readStringUntilEOF(str, *buf); return str; From 9936ee518186e420443cea5283250475fcb8b048 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 09:31:30 +0300 Subject: [PATCH 127/173] Update gtest_disk_encrypted for new readFile() interface --- src/Disks/tests/gtest_disk_encrypted.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/tests/gtest_disk_encrypted.cpp b/src/Disks/tests/gtest_disk_encrypted.cpp index 60cf8d81d92..85dd8eb78b8 100644 --- a/src/Disks/tests/gtest_disk_encrypted.cpp +++ b/src/Disks/tests/gtest_disk_encrypted.cpp @@ -57,7 +57,7 @@ protected: String getFileContents(const String & file_name) { - auto buf = encrypted_disk->readFile(file_name, {}, 0); + auto buf = encrypted_disk->readFile(file_name, /* settings= */ {}, /* size= */ {}); String str; readStringUntilEOF(str, *buf); return str; From 13850ba2570bb1d2c0a1598e527253db7a73bd93 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:01:29 +0300 Subject: [PATCH 128/173] tests/ci/docker_images_check: add missing time import --- tests/ci/docker_images_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 9bd3f431429..81c2ee6f911 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -5,6 +5,7 @@ from report import create_test_html_report from s3_helper import S3Helper import json import os +import time from pr_info import PRInfo from github import Github import shutil From fa50dd885c162ad2449335c68591d1bd007395c4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 129/173] Run pylint for ci scripts --- utils/check-style/check-style | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 7137d04a568..514f01ce4b9 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -70,7 +70,7 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.xml' | xargs xmllint --noout --nonet # FIXME: for now only clickhouse-test -pylint --rcfile=$ROOT_PATH/.pylintrc --persistent=no --score=n $ROOT_PATH/tests/clickhouse-test +pylint --rcfile=$ROOT_PATH/.pylintrc --persistent=no --score=n $ROOT_PATH/tests/clickhouse-test $ROOT_PATH/tests/ci/*.py find $ROOT_PATH -not -path $ROOT_PATH'/contrib*' \( -name '*.yaml' -or -name '*.yml' \) -type f | grep -vP $EXCLUDE_DIRS | From 944437caf68aff3c55409b05fd64e502b862bcaf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 130/173] Add PyGithub into style image --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 33cdb9db57a..0f58b0fd70a 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ pylint \ yamllint \ - && pip3 install codespell + && pip3 install codespell PyGithub COPY run.sh / COPY process_style_check_result.py / From c47617a6125f9a814fec1ce7717e726f4e3154cb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 131/173] tests/ci/build_check.py: fix pylint warnings --- tests/ci/build_check.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index ed1e3b534a9..6a88350f43c 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -2,16 +2,14 @@ # import subprocess import logging -from s3_helper import S3Helper import json import os -from pr_info import PRInfo -from github import Github -import shutil -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -import os import sys import time +from s3_helper import S3Helper +from pr_info import PRInfo +from github import Github +from get_robot_token import get_best_robot_token from version_helper import get_version_from_repo, update_version_local @@ -143,7 +141,7 @@ if __name__ == "__main__": for i in range(10): try: - logging.info(f"Pulling image {image_name}:{image_version}") + logging.info("Pulling image %s:%s", image_name, image_version) subprocess.check_output(f"docker pull {image_name}:{image_version}", stderr=subprocess.STDOUT, shell=True) break except Exception as ex: @@ -157,7 +155,7 @@ if __name__ == "__main__": update_version_local(repo_path, pr_info.sha, version) build_name = build_config_to_string(build_config) - logging.info(f"Build short name {build_name}") + logging.info("Build short name %s", build_name) subprocess.check_call(f"echo 'BUILD_NAME=build_urls_{build_name}' >> $GITHUB_ENV", shell=True) build_output_path = os.path.join(temp_path, build_name) From d1c49bbdcb5333e55625f345db2ba90090902029 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 132/173] tests/ci/build_report_check.py: fix pylint warnings --- tests/ci/build_report_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 4639365ded0..8bffe76d6b6 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import json +import logging +import os +import sys from github import Github from report import create_build_html_report from s3_helper import S3Helper -import logging -import os from get_robot_token import get_best_robot_token -import sys from pr_info import PRInfo -class BuildResult(object): +class BuildResult(): def __init__(self, compiler, build_type, sanitizer, bundled, splitted, status, elapsed_seconds, with_coverage): self.compiler = compiler self.build_type = build_type @@ -121,7 +121,7 @@ if __name__ == "__main__": branch_name = "PR #{}".format(pr_info.number) branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_info.number) commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{pr_info.sha}" - task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID', 0)}" + task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID', '0')}" report = create_build_html_report( build_check_name, build_results, From e581cdb933ccd97525c737b4ee8e46edba81b87b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 133/173] tests/ci/compress_files.py: fix pylint warnings --- tests/ci/compress_files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci/compress_files.py b/tests/ci/compress_files.py index f095b04872b..f3d2349408f 100644 --- a/tests/ci/compress_files.py +++ b/tests/ci/compress_files.py @@ -32,18 +32,18 @@ def compress_fast(path, archive_path, exclude=None): else: path += "/.." cmd = "tar {} {} -cf {} -C {} {}".format(pigz_part, exclude_part, archive_path, path, fname) - logging.debug("compress_fast cmd:{}".format(cmd)) + logging.debug("compress_fast cmd: %s", cmd) subprocess.check_call(cmd, shell=True) def decompress_fast(archive_path, result_path=None): pigz_part = '' if os.path.exists('/usr/bin/pigz'): - logging.info("pigz found, will compress and decompress faster ('{}' -> '{}')".format(archive_path, result_path)) + logging.info("pigz found, will compress and decompress faster ('%s' -> '%s')", archive_path, result_path) pigz_part = "--use-compress-program='pigz'" else: pigz_part = '-z' - logging.info("no pigz, decompressing with default tar ('{}' -> '{}')".format(archive_path, result_path)) + logging.info("no pigz, decompressing with default tar ('%s' -> '%s')", archive_path, result_path) if result_path is None: subprocess.check_call("tar {} -xf {}".format(pigz_part, archive_path), shell=True) From 2a3fe5f798f3d37f92fc4baec8dbfe654182088e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 134/173] tests/ci/docker_images_check.py: fix pylint warnings --- tests/ci/docker_images_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 81c2ee6f911..878689c0fb0 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 import subprocess import logging -from report import create_test_html_report -from s3_helper import S3Helper import json import os import time +import shutil +from report import create_test_html_report +from s3_helper import S3Helper from pr_info import PRInfo from github import Github -import shutil from get_robot_token import get_best_robot_token, get_parameter_from_ssm NAME = "Push to Dockerhub (actions)" @@ -57,7 +57,7 @@ def get_changed_docker_images(pr_info, repo_path, image_file_path): index += 1 if index > 100: # Sanity check to prevent infinite loop. - raise "Too many changed docker images, this is a bug." + str(changed_images) + raise RuntimeError("Too many changed docker images, this is a bug." + str(changed_images)) # If a dependent image was already in the list because its own files # changed, but then it was added as a dependent of a changed base, we @@ -205,7 +205,7 @@ if __name__ == "__main__": images_processing_result += process_single_image(versions, full_path, image_name) result_images[image_name] = pr_commit_version - if len(changed_images): + if changed_images: description = "Updated " + ','.join([im[1] for im in changed_images]) else: description = "Nothing to update" From 3dd09db096fdba2d8910939d9c8d458d1f658efc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 135/173] tests/ci/fast_test_check.py: fix pylint warnings --- tests/ci/fast_test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 5405187e2e5..47e1579fc74 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -5,12 +5,12 @@ import subprocess import os import json import time +import csv from pr_info import PRInfo from report import create_test_html_report from s3_helper import S3Helper from github import Github -from get_robot_token import get_best_robot_token, get_parameter_from_ssm -import csv +from get_robot_token import get_best_robot_token NAME = 'Fast test (actions)' From 0c9c15e06b119c8014b891ca9be4c5427240864c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:23 +0300 Subject: [PATCH 136/173] tests/ci/finish_check.py: fix pylint warnings --- tests/ci/finish_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index db405cf8f73..4833f62443c 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import logging -from github import Github -from pr_info import PRInfo import json import os +from github import Github +from pr_info import PRInfo from get_robot_token import get_best_robot_token NAME = 'Run Check (actions)' From b41c77f4f838563ff4e247fa06762e90350628ab Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:24 +0300 Subject: [PATCH 137/173] Add boto3 to style image --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 0f58b0fd70a..1b0daf2c20e 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ pylint \ yamllint \ - && pip3 install codespell PyGithub + && pip3 install codespell PyGithub boto3 COPY run.sh / COPY process_style_check_result.py / From 90ea2de9bb5deaaceb044474ad472242665438ed Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:24 +0300 Subject: [PATCH 138/173] Add unidiff into style image --- docker/test/style/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index 1b0daf2c20e..a9c1eca8de4 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ python3-pip \ pylint \ yamllint \ - && pip3 install codespell PyGithub boto3 + && pip3 install codespell PyGithub boto3 unidiff COPY run.sh / COPY process_style_check_result.py / From a90e477e412c921007a10684c76a6875d58aa662 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:24 +0300 Subject: [PATCH 139/173] tests/ci/pr_info.py: fix pylint warnings --- tests/ci/pr_info.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 5eb4cd6ad3e..0b4aeb56699 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -1,9 +1,6 @@ #!/usr/bin/env python3 -import requests -import json -import os -import subprocess import urllib +import requests from unidiff import PatchSet @@ -15,7 +12,7 @@ class PRInfo: else: self.sha = github_event['pull_request']['head']['sha'] - self.labels = set([l['name'] for l in github_event['pull_request']['labels']]) + self.labels = { l['name'] for l in github_event['pull_request']['labels'] } self.user_login = github_event['pull_request']['user']['login'] self.user_orgs = set([]) if need_orgs: @@ -27,9 +24,9 @@ class PRInfo: self.changed_files = set([]) if need_changed_files: diff_url = github_event['pull_request']['diff_url'] - diff = urllib.request.urlopen(github_event['pull_request']['diff_url']) + diff = urllib.request.urlopen(diff_url) diff_object = PatchSet(diff, diff.headers.get_charsets()[0]) - self.changed_files = set([f.path for f in diff_object]) + self.changed_files = { f.path for f in diff_object } def get_dict(self): return { From bcac58acdede3037611ce22a1be103c98455a0c1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:24 +0300 Subject: [PATCH 140/173] tests/ci/pvs_check.py: fix pylint warnings --- tests/ci/pvs_check.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/ci/pvs_check.py b/tests/ci/pvs_check.py index 03600476d92..96dfb01e234 100644 --- a/tests/ci/pvs_check.py +++ b/tests/ci/pvs_check.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 + +# pylint: disable=line-too-long + import subprocess import os import json import logging +import sys from github import Github from report import create_test_html_report from s3_helper import S3Helper from pr_info import PRInfo -import shutil -import sys from get_robot_token import get_best_robot_token, get_parameter_from_ssm NAME = 'PVS Studio (actions)' @@ -34,10 +36,12 @@ def _process_txt_report(path): for line in report_file: if 'viva64' in line: continue - elif 'warn' in line: + + if 'warn' in line: warnings.append(':'.join(line.split('\t')[0:2])) elif 'err' in line: errors.append(':'.join(line.split('\t')[0:2])) + return warnings, errors def get_commit(gh, commit_sha): From 4e2ffbd38c8342af501c82ac73caf84dd81bff96 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:24 +0300 Subject: [PATCH 141/173] tests/ci/report.py: fix pylint warnings --- tests/ci/report.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/ci/report.py b/tests/ci/report.py index a102a005d9b..156e6096605 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -128,7 +128,10 @@ def _get_html_url(url): return '' -def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=[], with_raw_logs=False): +def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=None, with_raw_logs=False): + if additional_urls is None: + additional_urls = [] + if test_result: rows_part = "" num_fails = 0 @@ -149,7 +152,7 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u has_test_logs = True row = "" - is_fail = test_status == "FAIL" or test_status == 'FLAKY' + is_fail = test_status in ('FAIL', 'FLAKY') if is_fail and with_raw_logs and test_logs is not None: row = "" row += "" + test_name + "" From ce7e18fcb7a79c826d42eee9963c1cc121e1bb3e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:24 +0300 Subject: [PATCH 142/173] tests/ci/run_check.py: fix pylint warnings --- tests/ci/run_check.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 106f005663f..3d81625b838 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 import os import json -import requests -from pr_info import PRInfo import sys import logging +from pr_info import PRInfo from github import Github from get_robot_token import get_best_robot_token @@ -70,16 +69,16 @@ TRUSTED_CONTRIBUTORS = { def pr_is_by_trusted_user(pr_user_login, pr_user_orgs): if pr_user_login in TRUSTED_CONTRIBUTORS: - logging.info("User '{}' is trusted".format(pr_user_login)) + logging.info("User '%s' is trusted", pr_user_login) return True - logging.info("User '{}' is not trusted".format(pr_user_login)) + logging.info("User '%s' is not trusted", pr_user_login) for org_id in pr_user_orgs: if org_id in TRUSTED_ORG_IDS: - logging.info("Org '{}' is trusted; will mark user {} as trusted".format(org_id, pr_user_login)) + logging.info("Org '%s' is trusted; will mark user %s as trusted", org_id, pr_user_login) return True - logging.info("Org '{}' is not trusted".format(org_id)) + logging.info("Org '%s' is not trusted", org_id) return False From e933e0566290b18c7cf0b99e4ebf8c7937e8cb53 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:25 +0300 Subject: [PATCH 143/173] tests/ci/s3_helper.py: fix pylint warnings --- tests/ci/s3_helper.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index ed0e66ee7a1..292aa5c0472 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -2,18 +2,16 @@ import hashlib import logging import os -import boto3 -from botocore.exceptions import ClientError, BotoCoreError from multiprocessing.dummy import Pool +import boto3 from compress_files import compress_file_fast -from get_robot_token import get_parameter_from_ssm def _md5(fname): hash_md5 = hashlib.md5() with open(fname, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) - logging.debug("MD5 for {} is {}".format(fname, hash_md5.hexdigest())) + logging.debug("MD5 for %s is %s", fname, hash_md5.hexdigest()) return hash_md5.hexdigest() @@ -27,13 +25,13 @@ def _flatten_list(lst): return result -class S3Helper(object): +class S3Helper(): def __init__(self, host): self.session = boto3.session.Session(region_name='us-east-1') self.client = self.session.client('s3', endpoint_url=host) def _upload_file_to_s3(self, bucket_name, file_path, s3_path): - logging.debug("Start uploading {} to bucket={} path={}".format(file_path, bucket_name, s3_path)) + logging.debug("Start uploading %s to bucket=%s path=%s", file_path, bucket_name, s3_path) metadata = {} if os.path.getsize(file_path) < 64 * 1024 * 1024: if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"): @@ -55,7 +53,7 @@ class S3Helper(object): logging.info("File is too large, do not provide content type") self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata) - logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata)) + logging.info("Upload %s to %s. Meta: %s", file_path, s3_path, metadata) # last two replacements are specifics of AWS urls: https://jamesd3142.wordpress.com/2018/02/28/amazon-s3-and-the-plus-symbol/ return "https://s3.amazonaws.com/{bucket}/{path}".format(bucket=bucket_name, path=s3_path).replace('+', '%2B').replace(' ', '%20') @@ -66,7 +64,7 @@ class S3Helper(object): return self._upload_file_to_s3('clickhouse-builds', file_path, s3_path) def _upload_folder_to_s3(self, folder_path, s3_folder_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks): - logging.info("Upload folder '{}' to bucket={} of s3 folder '{}'".format(folder_path, bucket_name, s3_folder_path)) + logging.info("Upload folder '%s' to bucket=%s of s3 folder '%s'", folder_path, bucket_name, s3_folder_path) if not os.path.exists(folder_path): return [] files = os.listdir(folder_path) From 161f48c2c84378a91b6cee83c72e6bc1ead4b045 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:25 +0300 Subject: [PATCH 144/173] tests/ci/style_check.py: fix pylint warnings --- tests/ci/style_check.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 147909516fa..a0516623c41 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 -from github import Github -from report import create_test_html_report -import shutil import logging import subprocess import os import csv -from s3_helper import S3Helper import time import json +from github import Github +from report import create_test_html_report +from s3_helper import S3Helper from pr_info import PRInfo from get_robot_token import get_best_robot_token From 5930714ccdf9fa9c05a7d34b2d8d730c0904f633 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:03:25 +0300 Subject: [PATCH 145/173] tests/ci/version_helper.py: fix pylint warnings --- tests/ci/version_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 9c89b4df65e..dd3845eae66 100644 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -9,7 +9,7 @@ CHANGELOG_PATH = "debian/changelog" CONTRIBUTORS_SCRIPT_DIR = "src/Storages/System/" -class ClickHouseVersion(object): +class ClickHouseVersion(): def __init__(self, major, minor, patch, tweak, revision): self.major = major self.minor = minor @@ -53,7 +53,7 @@ class ClickHouseVersion(object): return (self.major, self.minor, self.patch, self.tweak) -class VersionType(object): +class VersionType(): STABLE = "stable" TESTING = "testing" From d8a64a7ecefea2aa012139ed69b9796089cad874 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 27 Oct 2021 10:14:39 +0300 Subject: [PATCH 146/173] tests/ci: fix wrong-import-order pylint warning --- tests/ci/build_check.py | 2 +- tests/ci/docker_images_check.py | 2 +- tests/ci/fast_test_check.py | 2 +- tests/ci/run_check.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 6a88350f43c..761bcb736a3 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -6,9 +6,9 @@ import json import os import sys import time +from github import Github from s3_helper import S3Helper from pr_info import PRInfo -from github import Github from get_robot_token import get_best_robot_token from version_helper import get_version_from_repo, update_version_local diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index 878689c0fb0..470dcc18233 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -5,10 +5,10 @@ import json import os import time import shutil +from github import Github from report import create_test_html_report from s3_helper import S3Helper from pr_info import PRInfo -from github import Github from get_robot_token import get_best_robot_token, get_parameter_from_ssm NAME = "Push to Dockerhub (actions)" diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index 47e1579fc74..19dd3a6d1aa 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -6,10 +6,10 @@ import os import json import time import csv +from github import Github from pr_info import PRInfo from report import create_test_html_report from s3_helper import S3Helper -from github import Github from get_robot_token import get_best_robot_token NAME = 'Fast test (actions)' diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 3d81625b838..2af50c895da 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -3,8 +3,8 @@ import os import json import sys import logging -from pr_info import PRInfo from github import Github +from pr_info import PRInfo from get_robot_token import get_best_robot_token NAME = 'Run Check (actions)' From 290bd317fb63f04b3607f1650f6f96cce9bdee7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1ty=C3=A1s=20Jani?= Date: Wed, 27 Oct 2021 09:49:54 +0200 Subject: [PATCH 147/173] Update documentation for TSVWithNames parsing The documentation of the format TabSeparatedWithNames stated during parsing the first row (header) is ignored. It is not true anymore, the header is parsed, and the column position is determined from the header. --- docs/en/interfaces/formats.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index d0e5c44b4f7..8fc031834be 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -164,8 +164,7 @@ This format is also available under the name `TSVRaw`. ## TabSeparatedWithNames {#tabseparatedwithnames} Differs from the `TabSeparated` format in that the column names are written in the first row. -During parsing, the first row is completely ignored. You can’t use column names to determine their position or to check their correctness. -(Support for parsing the header row may be added in the future.) +During parsing, the first row is expected to contain the column names. You can use column names to determine their position and to check their correctness. This format is also available under the name `TSVWithNames`. From fd002e327cc102ad167f93e9452fce977d219c5a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 27 Oct 2021 11:26:42 +0300 Subject: [PATCH 148/173] Updated HashedArrayDictionary --- src/Dictionaries/HashedArrayDictionary.cpp | 86 +--------------------- 1 file changed, 2 insertions(+), 84 deletions(-) diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 94145e8fa99..a92f8bc1191 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -55,95 +55,13 @@ ColumnPtr HashedArrayDictionary::getColumn( DictionaryKeysArenaHolder arena_holder; DictionaryKeysExtractor extractor(key_columns, arena_holder.getComplexKeyArena()); - const size_t size = extractor.getKeysSize(); + const size_t keys_size = extractor.getKeysSize(); const auto & dictionary_attribute = dict_struct.getAttribute(attribute_name, result_type); const size_t attribute_index = dict_struct.attribute_name_to_index.find(attribute_name)->second; auto & attribute = attributes[attribute_index]; - bool is_attribute_nullable = attribute.is_index_null.has_value(); - - ColumnUInt8::MutablePtr col_null_map_to; - ColumnUInt8::Container * vec_null_map_to = nullptr; - if (attribute.is_index_null) - { - col_null_map_to = ColumnUInt8::create(size, false); - vec_null_map_to = &col_null_map_to->getData(); - } - - auto type_call = [&](const auto & dictionary_attribute_type) - { - using Type = std::decay_t; - using AttributeType = typename Type::AttributeType; - using ValueType = DictionaryValueType; - using ColumnProvider = DictionaryAttributeColumnProvider; - - DictionaryDefaultValueExtractor default_value_extractor(dictionary_attribute.null_value, default_values_column); - - auto column = ColumnProvider::getColumn(dictionary_attribute, size); - - if constexpr (std::is_same_v) - { - auto * out = column.get(); - - getItemsImpl( - attribute, - extractor, - [&](const size_t, const Array & value, bool) { out->insert(value); }, - default_value_extractor); - } - else if constexpr (std::is_same_v) - { - auto * out = column.get(); - - if (is_attribute_nullable) - getItemsImpl( - attribute, - extractor, - [&](size_t row, const StringRef value, bool is_null) - { - (*vec_null_map_to)[row] = is_null; - out->insertData(value.data, value.size); - }, - default_value_extractor); - else - getItemsImpl( - attribute, - extractor, - [&](size_t, const StringRef value, bool) { out->insertData(value.data, value.size); }, - default_value_extractor); - } - else - { - auto & out = column->getData(); - - if (is_attribute_nullable) - getItemsImpl( - attribute, - extractor, - [&](size_t row, const auto value, bool is_null) - { - (*vec_null_map_to)[row] = is_null; - out[row] = value; - }, - default_value_extractor); - else - getItemsImpl( - attribute, - extractor, - [&](size_t row, const auto value, bool) { out[row] = value; }, - default_value_extractor); - } - - result = std::move(column); - }; - - callOnDictionaryAttributeType(attribute.type, type_call); - - if (is_attribute_nullable) - result = ColumnNullable::create(std::move(result), std::move(col_null_map_to)); - - return result; + return getAttributeColumn(attribute, dictionary_attribute, keys_size, default_values_column, extractor); } template From d523b284593b2c1fac182fc06af134dcc95e3116 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 27 Oct 2021 11:28:21 +0300 Subject: [PATCH 149/173] SQLUserDefinedFunctions support ON CLUSTER --- .../InterpreterCreateFunctionQuery.cpp | 15 ++++++--- .../InterpreterDropFunctionQuery.cpp | 32 +++++++++++++----- src/Parsers/ASTCreateFunctionQuery.cpp | 6 ++++ src/Parsers/ASTCreateFunctionQuery.h | 9 +++-- src/Parsers/ASTDropFunctionQuery.cpp | 1 + src/Parsers/ASTDropFunctionQuery.h | 8 +++-- src/Parsers/ParserCreateFunctionQuery.cpp | 9 +++++ src/Parsers/ParserDropFunctionQuery.cpp | 12 ++++++- .../__init__.py | 0 .../configs/config.d/clusters.xml | 22 +++++++++++++ .../test.py | 33 +++++++++++++++++++ 11 files changed, 128 insertions(+), 19 deletions(-) create mode 100644 tests/integration/test_sql_user_defined_functions_on_cluster/__init__.py create mode 100644 tests/integration/test_sql_user_defined_functions_on_cluster/configs/config.d/clusters.xml create mode 100644 tests/integration/test_sql_user_defined_functions_on_cluster/test.py diff --git a/src/Interpreters/InterpreterCreateFunctionQuery.cpp b/src/Interpreters/InterpreterCreateFunctionQuery.cpp index fe331985aa4..a995eed114f 100644 --- a/src/Interpreters/InterpreterCreateFunctionQuery.cpp +++ b/src/Interpreters/InterpreterCreateFunctionQuery.cpp @@ -1,7 +1,5 @@ #include -#include - #include #include #include @@ -11,6 +9,7 @@ #include #include #include +#include namespace DB @@ -24,15 +23,21 @@ namespace ErrorCodes BlockIO InterpreterCreateFunctionQuery::execute() { - auto current_context = getContext(); - current_context->checkAccess(AccessType::CREATE_FUNCTION); - FunctionNameNormalizer().visit(query_ptr.get()); auto * create_function_query = query_ptr->as(); if (!create_function_query) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Expected CREATE FUNCTION query"); + AccessRightsElements access_rights_elements; + access_rights_elements.emplace_back(AccessType::CREATE_FUNCTION); + + if (!create_function_query->cluster.empty()) + return executeDDLQueryOnCluster(query_ptr, getContext(), access_rights_elements); + + auto current_context = getContext(); + current_context->checkAccess(access_rights_elements); + auto & user_defined_function_factory = UserDefinedSQLFunctionFactory::instance(); auto & function_name = create_function_query->function_name; diff --git a/src/Interpreters/InterpreterDropFunctionQuery.cpp b/src/Interpreters/InterpreterDropFunctionQuery.cpp index b788c8f960f..5cc32d30f24 100644 --- a/src/Interpreters/InterpreterDropFunctionQuery.cpp +++ b/src/Interpreters/InterpreterDropFunctionQuery.cpp @@ -1,30 +1,46 @@ +#include + #include #include #include #include #include #include -#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int UNSUPPORTED_METHOD; +} + BlockIO InterpreterDropFunctionQuery::execute() { - auto current_context = getContext(); - current_context->checkAccess(AccessType::DROP_FUNCTION); - FunctionNameNormalizer().visit(query_ptr.get()); - auto & drop_function_query = query_ptr->as(); + auto * drop_function_query = query_ptr->as(); + + if (!drop_function_query) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Expected DROP FUNCTION query"); + + AccessRightsElements access_rights_elements; + access_rights_elements.emplace_back(AccessType::DROP_FUNCTION); + + if (!drop_function_query->cluster.empty()) + return executeDDLQueryOnCluster(query_ptr, getContext(), access_rights_elements); + + auto current_context = getContext(); + current_context->checkAccess(access_rights_elements); auto & user_defined_functions_factory = UserDefinedSQLFunctionFactory::instance(); - if (drop_function_query.if_exists && !user_defined_functions_factory.has(drop_function_query.function_name)) + if (drop_function_query->if_exists && !user_defined_functions_factory.has(drop_function_query->function_name)) return {}; - UserDefinedSQLFunctionFactory::instance().unregisterFunction(drop_function_query.function_name); - UserDefinedSQLObjectsLoader::instance().removeObject(current_context, UserDefinedSQLObjectType::Function, drop_function_query.function_name); + UserDefinedSQLFunctionFactory::instance().unregisterFunction(drop_function_query->function_name); + UserDefinedSQLObjectsLoader::instance().removeObject(current_context, UserDefinedSQLObjectType::Function, drop_function_query->function_name); return {}; } diff --git a/src/Parsers/ASTCreateFunctionQuery.cpp b/src/Parsers/ASTCreateFunctionQuery.cpp index 4e1e7de660d..9209b1f1869 100644 --- a/src/Parsers/ASTCreateFunctionQuery.cpp +++ b/src/Parsers/ASTCreateFunctionQuery.cpp @@ -1,6 +1,9 @@ #include #include #include +#include +#include + namespace DB { @@ -25,6 +28,9 @@ void ASTCreateFunctionQuery::formatImpl(const IAST::FormatSettings & settings, I settings.ostr << (settings.hilite ? hilite_none : ""); settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : ""); + + formatOnCluster(settings); + settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); function_core->formatImpl(settings, state, frame); } diff --git a/src/Parsers/ASTCreateFunctionQuery.h b/src/Parsers/ASTCreateFunctionQuery.h index a58fe64c435..fccc1837135 100644 --- a/src/Parsers/ASTCreateFunctionQuery.h +++ b/src/Parsers/ASTCreateFunctionQuery.h @@ -1,12 +1,13 @@ #pragma once -#include -#include +#include +#include + namespace DB { -class ASTCreateFunctionQuery : public IAST +class ASTCreateFunctionQuery : public IAST, public ASTQueryWithOnCluster { public: String function_name; @@ -20,6 +21,8 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTDropFunctionQuery.cpp b/src/Parsers/ASTDropFunctionQuery.cpp index 47665aa52f9..3409d699042 100644 --- a/src/Parsers/ASTDropFunctionQuery.cpp +++ b/src/Parsers/ASTDropFunctionQuery.cpp @@ -19,6 +19,7 @@ void ASTDropFunctionQuery::formatImpl(const IAST::FormatSettings & settings, IAS settings.ostr << (settings.hilite ? hilite_none : ""); settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : ""); + formatOnCluster(settings); } } diff --git a/src/Parsers/ASTDropFunctionQuery.h b/src/Parsers/ASTDropFunctionQuery.h index a9d70a3016f..c9f673578a9 100644 --- a/src/Parsers/ASTDropFunctionQuery.h +++ b/src/Parsers/ASTDropFunctionQuery.h @@ -1,11 +1,13 @@ #pragma once -#include "IAST.h" +#include +#include + namespace DB { -class ASTDropFunctionQuery : public IAST +class ASTDropFunctionQuery : public IAST, public ASTQueryWithOnCluster { public: String function_name; @@ -17,6 +19,8 @@ public: ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; + + ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ParserCreateFunctionQuery.cpp b/src/Parsers/ParserCreateFunctionQuery.cpp index 5d84b6bc2dc..55e6e2df1d2 100644 --- a/src/Parsers/ParserCreateFunctionQuery.cpp +++ b/src/Parsers/ParserCreateFunctionQuery.cpp @@ -17,6 +17,7 @@ bool ParserCreateFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp ParserKeyword s_function("FUNCTION"); ParserKeyword s_or_replace("OR REPLACE"); ParserKeyword s_if_not_exists("IF NOT EXISTS"); + ParserKeyword s_on("ON"); ParserIdentifier function_name_p; ParserKeyword s_as("AS"); ParserLambdaExpression lambda_p; @@ -24,6 +25,7 @@ bool ParserCreateFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp ASTPtr function_name; ASTPtr function_core; + String cluster_str; bool or_replace = false; bool if_not_exists = false; @@ -42,6 +44,12 @@ bool ParserCreateFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp if (!function_name_p.parse(pos, function_name, expected)) return false; + if (s_on.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + if (!s_as.ignore(pos, expected)) return false; @@ -55,6 +63,7 @@ bool ParserCreateFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp create_function_query->function_core = function_core; create_function_query->or_replace = or_replace; create_function_query->if_not_exists = if_not_exists; + create_function_query->cluster = std::move(cluster_str); return true; } diff --git a/src/Parsers/ParserDropFunctionQuery.cpp b/src/Parsers/ParserDropFunctionQuery.cpp index d8c86646410..3f35124f9ab 100644 --- a/src/Parsers/ParserDropFunctionQuery.cpp +++ b/src/Parsers/ParserDropFunctionQuery.cpp @@ -12,8 +12,10 @@ bool ParserDropFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expec ParserKeyword s_drop("DROP"); ParserKeyword s_function("FUNCTION"); ParserKeyword s_if_exists("IF EXISTS"); - + ParserKeyword s_on("ON"); ParserIdentifier function_name_p; + + String cluster_str; bool if_exists = false; ASTPtr function_name; @@ -30,8 +32,16 @@ bool ParserDropFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expec if (!function_name_p.parse(pos, function_name, expected)) return false; + if (s_on.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + auto drop_function_query = std::make_shared(); drop_function_query->if_exists = if_exists; + drop_function_query->cluster = std::move(cluster_str); + node = drop_function_query; drop_function_query->function_name = function_name->as().name(); diff --git a/tests/integration/test_sql_user_defined_functions_on_cluster/__init__.py b/tests/integration/test_sql_user_defined_functions_on_cluster/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_sql_user_defined_functions_on_cluster/configs/config.d/clusters.xml b/tests/integration/test_sql_user_defined_functions_on_cluster/configs/config.d/clusters.xml new file mode 100644 index 00000000000..4d668f591b9 --- /dev/null +++ b/tests/integration/test_sql_user_defined_functions_on_cluster/configs/config.d/clusters.xml @@ -0,0 +1,22 @@ + + + + + + ch1 + 9000 + + + ch2 + 9000 + + + + + ch3 + 9000 + + + + + diff --git a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py new file mode 100644 index 00000000000..48f5733b9e4 --- /dev/null +++ b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py @@ -0,0 +1,33 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance('ch1', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) +ch2 = cluster.add_instance('ch2', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) +ch3 = cluster.add_instance('ch3', main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_sql_user_defined_functions_on_cluster(): + assert "Unknown function test_function" in ch1.query_and_get_error("SELECT test_function(1);") + assert "Unknown function test_function" in ch2.query_and_get_error("SELECT test_function(1);") + assert "Unknown function test_function" in ch3.query_and_get_error("SELECT test_function(1);") + + ch1.query_with_retry("CREATE FUNCTION test_function ON CLUSTER 'cluster' AS x -> x + 1;", retry_count=5) + assert ch1.query("SELECT test_function(1);") == "2\n" + assert ch2.query("SELECT test_function(1);") == "2\n" + assert ch3.query("SELECT test_function(1);") == "2\n" + + ch2.query_with_retry("DROP FUNCTION test_function ON CLUSTER 'cluster'", retry_count=5) + assert "Unknown function test_function" in ch1.query_and_get_error("SELECT test_function(1);") + assert "Unknown function test_function" in ch2.query_and_get_error("SELECT test_function(1);") + assert "Unknown function test_function" in ch3.query_and_get_error("SELECT test_function(1);") From 9e5337b5d4ac7e0e747f2644ced94285ee72301f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 27 Oct 2021 11:51:47 +0300 Subject: [PATCH 150/173] DictionarySource unknown column name in dictionary fix --- src/Dictionaries/DictionarySource.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Dictionaries/DictionarySource.cpp b/src/Dictionaries/DictionarySource.cpp index 1f08a83fbba..c3f0ecf3cde 100644 --- a/src/Dictionaries/DictionarySource.cpp +++ b/src/Dictionaries/DictionarySource.cpp @@ -7,6 +7,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int UNSUPPORTED_METHOD; } bool DictionarySourceCoordinator::getKeyColumnsNextRangeToRead(ColumnsWithTypeAndName & key_columns, ColumnsWithTypeAndName & data_columns) @@ -56,7 +57,7 @@ void DictionarySourceCoordinator::initialize(const Names & column_names) { column_with_type.type = dictionary_structure.range_max->type; } - else + else if (dictionary_structure.key.has_value()) { const auto & dictionary_key_attributes = *dictionary_structure.key; for (const auto & attribute : dictionary_key_attributes) @@ -68,6 +69,12 @@ void DictionarySourceCoordinator::initialize(const Names & column_names) } } } + else + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "No such column name {} in dictionary {}", + column_name, + dictionary->getDictionaryID().getNameForLogs()); + } } else { From b96bd7ee4268e3eb15904da10f3bc329d77556d1 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Oct 2021 12:29:43 +0300 Subject: [PATCH 151/173] Update 01083_expressions_in_engine_arguments.sql --- .../01083_expressions_in_engine_arguments.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index bd4712cca21..bd652bbe8a1 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -1,5 +1,15 @@ -- Tags: no-parallel, no-fasttest +DROP TABLE IF EXISTS file; +DROP TABLE IF EXISTS url; +DROP TABLE IF EXISTS view; +DROP TABLE IF EXISTS buffer; +DROP TABLE IF EXISTS merge; +DROP TABLE IF EXISTS merge_tf; +DROP TABLE IF EXISTS distributed; +DROP TABLE IF EXISTS distributed_tf; +DROP DICTIONARY IF EXISTS dict; + CREATE TABLE file (n Int8) ENGINE = File(upper('tsv') || 'WithNames' || 'AndTypes'); CREATE TABLE buffer (n Int8) ENGINE = Buffer(currentDatabase(), file, 16, 10, 200, 10000, 1000000, 10000000, 1000000000); CREATE TABLE merge (n Int8) ENGINE = Merge('', lower('DISTRIBUTED')); From a2fd9214656abbef454497310a77d6533454cfc4 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Oct 2021 12:34:20 +0300 Subject: [PATCH 152/173] Update 01083_expressions_in_engine_arguments.sql --- .../0_stateless/01083_expressions_in_engine_arguments.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index bd652bbe8a1..6c7b3936e07 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -8,6 +8,7 @@ DROP TABLE IF EXISTS merge; DROP TABLE IF EXISTS merge_tf; DROP TABLE IF EXISTS distributed; DROP TABLE IF EXISTS distributed_tf; +DROP TABLE IF EXISTS rich_syntax; DROP DICTIONARY IF EXISTS dict; CREATE TABLE file (n Int8) ENGINE = File(upper('tsv') || 'WithNames' || 'AndTypes'); From 996706101dfd408473ca3554487ad5d59758f914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=8C=E6=B6=9B?= Date: Wed, 27 Oct 2021 15:14:42 +0800 Subject: [PATCH 153/173] optimized code --- src/Functions/MatchImpl.h | 5 ----- src/Functions/like.cpp | 5 +---- src/Functions/like.h | 17 +++++++++++++++++ src/Functions/map.cpp | 19 +++---------------- 4 files changed, 21 insertions(+), 25 deletions(-) create mode 100644 src/Functions/like.h diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 4849fe6e12d..7dc0712023f 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -419,9 +419,4 @@ struct MatchImpl } }; -struct NameLike -{ - static constexpr auto name = "like"; -}; - } diff --git a/src/Functions/like.cpp b/src/Functions/like.cpp index 5d2d5de1be2..bd73aa4d471 100644 --- a/src/Functions/like.cpp +++ b/src/Functions/like.cpp @@ -1,14 +1,11 @@ #include "FunctionsStringSearch.h" #include "FunctionFactory.h" -#include "MatchImpl.h" +#include "like.h" namespace DB { -using LikeImpl = MatchImpl; -using FunctionLike = FunctionsStringSearch; - void registerFunctionLike(FunctionFactory & factory) { factory.registerFunction(); diff --git a/src/Functions/like.h b/src/Functions/like.h new file mode 100644 index 00000000000..a00891ec64c --- /dev/null +++ b/src/Functions/like.h @@ -0,0 +1,17 @@ +#pragma once + +#include "MatchImpl.h" +#include "FunctionsStringSearch.h" + +namespace DB +{ + +struct NameLike +{ + static constexpr auto name = "like"; +}; + +using LikeImpl = MatchImpl; +using FunctionLike = FunctionsStringSearch; + +} diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index f84312ba6dc..edb0c28c980 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -16,7 +16,7 @@ #include #include #include "array/arrayIndex.h" -#include "Functions/MatchImpl.h" +#include "Functions/like.h" #include "Functions/FunctionsStringSearch.h" @@ -31,9 +31,6 @@ namespace ErrorCodes namespace { -using LikeImpl = MatchImpl; -using FunctionLike = FunctionsStringSearch; - // map(x, y, ...) is a function that allows you to make key-value pair class FunctionMap : public IFunction { @@ -347,20 +344,10 @@ public: }; auto res = func_like.executeImpl(new_arguments, result_type, input_rows_count); - const auto & container = checkAndGetColumn(res.get())->getData(); - bool exist = 0; - for (auto iter = container.begin(); iter != container.end(); iter++) - { - if (*iter == 1) - { - exist = 1; - break; - } - } - - vec_res[row] = exist; + const auto it = std::find_if(container.begin(), container.end(), [](int element){ return element == 1; }); // NOLINT + vec_res[row] = it == container.end() ? 0 : 1; } return col_res; From 4d8f913af55c500bae53ce1ed18537ad4b7720d1 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:00:16 +0300 Subject: [PATCH 154/173] Update 01083_expressions_in_engine_arguments.sql --- .../01083_expressions_in_engine_arguments.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 6c7b3936e07..43a0ba62d79 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -84,3 +84,14 @@ SELECT sum(n) from rich_syntax; -- Clear cache to avoid future errors in the logs SYSTEM DROP DNS CACHE + +DROP TABLE IF EXISTS file; +DROP TABLE IF EXISTS url; +DROP TABLE IF EXISTS view; +DROP TABLE IF EXISTS buffer; +DROP TABLE IF EXISTS merge; +DROP TABLE IF EXISTS merge_tf; +DROP TABLE IF EXISTS distributed; +DROP TABLE IF EXISTS distributed_tf; +DROP TABLE IF EXISTS rich_syntax; +DROP DICTIONARY IF EXISTS dict; From 8f8423aecce9ee9cecc361f59dfd5c07f1fdb47c Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:04:15 +0300 Subject: [PATCH 155/173] Update 01083_expressions_in_engine_arguments.sql --- .../01083_expressions_in_engine_arguments.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 43a0ba62d79..9a29117ff5d 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -85,13 +85,13 @@ SELECT sum(n) from rich_syntax; -- Clear cache to avoid future errors in the logs SYSTEM DROP DNS CACHE -DROP TABLE IF EXISTS file; -DROP TABLE IF EXISTS url; -DROP TABLE IF EXISTS view; -DROP TABLE IF EXISTS buffer; -DROP TABLE IF EXISTS merge; -DROP TABLE IF EXISTS merge_tf; -DROP TABLE IF EXISTS distributed; -DROP TABLE IF EXISTS distributed_tf; -DROP TABLE IF EXISTS rich_syntax; -DROP DICTIONARY IF EXISTS dict; +DROP TABLE file; +DROP TABLE url; +DROP TABLE view; +DROP TABLE buffer; +DROP TABLE merge; +DROP TABLE merge_tf; +DROP TABLE distributed; +DROP TABLE distributed_tf; +DROP TABLE rich_syntax; +DROP DICTIONARY dict; From 5e622d4c9850b794af74d32b193dc8c07ce4d188 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 27 Oct 2021 15:59:26 +0300 Subject: [PATCH 156/173] fix another issue --- .../ReplicatedMergeTreePartCheckThread.cpp | 23 +++++++++++++++++++ .../ReplicatedMergeTreePartCheckThread.h | 2 ++ src/Storages/StorageReplicatedMergeTree.cpp | 3 +++ 3 files changed, 28 insertions(+) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 7df7d45ff3c..8d3cb146990 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -67,6 +67,29 @@ void ReplicatedMergeTreePartCheckThread::enqueuePart(const String & name, time_t task->schedule(); } +void ReplicatedMergeTreePartCheckThread::cancelRemovedPartsCheck(const MergeTreePartInfo & drop_range_info) +{ + /// Wait for running tasks to finish and temporarily stop checking + stop(); + SCOPE_EXIT({ start(); }); + { + std::lock_guard lock(parts_mutex); + for (auto it = parts_queue.begin(); it != parts_queue.end();) + { + if (drop_range_info.contains(MergeTreePartInfo::fromPartName(it->first, storage.format_version))) + { + /// Remove part from the queue to avoid part resurrection + /// if we will check it and enqueue fetch after DROP/REPLACE execution. + parts_set.erase(it->first); + it = parts_queue.erase(it); + } + else + { + ++it; + } + } + } +} size_t ReplicatedMergeTreePartCheckThread::size() const { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h index e9366d7fb12..a8ce4fedd6d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h @@ -71,6 +71,8 @@ public: /// Check part by name CheckResult checkPart(const String & part_name); + void cancelRemovedPartsCheck(const MergeTreePartInfo & drop_range_info); + private: void run(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 053020f7007..02fb910b1a0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1769,8 +1769,10 @@ bool StorageReplicatedMergeTree::executeFetchShared( void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) { + LOG_TRACE(log, "Executing DROP_RANGE {}", entry.new_part_name); auto drop_range_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); getContext()->getMergeList().cancelInPartition(getStorageID(), drop_range_info.partition_id, drop_range_info.max_block); + part_check_thread.cancelRemovedPartsCheck(drop_range_info); queue.removePartProducingOpsInRange(getZooKeeper(), drop_range_info, entry); /// Delete the parts contained in the range to be deleted. @@ -1838,6 +1840,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (replace) { getContext()->getMergeList().cancelInPartition(getStorageID(), drop_range.partition_id, drop_range.max_block); + part_check_thread.cancelRemovedPartsCheck(drop_range); queue.removePartProducingOpsInRange(getZooKeeper(), drop_range, entry); } else From 456d548b6b3eb6e07df5492fd24ad74efe82a9ab Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Wed, 27 Oct 2021 16:17:56 +0300 Subject: [PATCH 157/173] Apply suggestions from code review Co-authored-by: Anton Popov --- docs/en/operations/settings/settings.md | 8 ++++---- docs/ru/operations/settings/settings.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2287db1b6c0..7fbe1cbe5c8 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3845,12 +3845,12 @@ Default value: `16`. ## wait_for_async_insert {#wait-for-async-insert} -Enables or disables waiting for processing of asynchronous insertion. If enabled, client will return `OK` only after the data is inserted. Otherwise, it will return `OK` even if the data wasn't inserted. +Enables or disables waiting for processing of asynchronous insertion. If enabled, server will return `OK` only after the data is inserted. Otherwise, it will return `OK` even if the data wasn't inserted. Possible values: -- 0 — Client returns `OK` even if the data is not yet inserted. -- 1 — Client returns `OK` only after the data is inserted. +- 0 — Server returns `OK` even if the data is not yet inserted. +- 1 — Server returns `OK` only after the data is inserted. Default value: `1`. @@ -3867,7 +3867,7 @@ Default value: [lock_acquire_timeout](#lock_acquire_timeout). ## async_insert_max_data_size {#async-insert-max-data-size} -The maximum size of the uparsed data in bytes collected per query before being inserted. +The maximum size of the unparsed data in bytes collected per query before being inserted. Possible values: diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index ffe52d365d3..fc0b2dd7fc3 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3675,8 +3675,8 @@ SELECT * FROM positional_arguments ORDER BY 2,3; Возможные значения: -- 0 — клиент возвращает `OK` даже если вставка данных еще не завершена. -- 1 — клиент возвращает `OK` только после завершения вставки данных. +- 0 — сервер возвращает `OK` даже если вставка данных еще не завершена. +- 1 — сервер возвращает `OK` только после завершения вставки данных. Значение по умолчанию: `1`. From 527474da643118ff2a8c0303aa5d8e378eec3482 Mon Sep 17 00:00:00 2001 From: gyuton Date: Wed, 27 Oct 2021 16:21:50 +0300 Subject: [PATCH 158/173] Fix wording --- docs/en/operations/settings/settings.md | 2 +- docs/en/sql-reference/statements/insert-into.md | 2 +- docs/ru/operations/settings/settings.md | 2 +- docs/ru/sql-reference/statements/insert-into.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 7fbe1cbe5c8..b8bc8656f44 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3817,7 +3817,7 @@ Default value: `0`. ## async_insert {#async-insert} -Enables or disables asynchronous inserts for multiple concurrent connections. This makes sense only for insertion over HTTP protocol. Note that deduplication isn't working for such inserts. +Enables or disables asynchronous inserts. This makes sense only for insertion over HTTP protocol. Note that deduplication isn't working for such inserts. If enabled, the data is combined into batches before the insertion into tables, so it is possible to do small and frequent insertions into ClickHouse (up to 15000 queries per second) without buffer tables. diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 0a37a5f41ae..cc166686793 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -119,7 +119,7 @@ Performance will not decrease if: - Data is added in real time. - You upload data that is usually sorted by time. -It's also possible to asynchronously insert data in small but frequent inserts from multiple concurrent connections. The data from such insertions is combined into batches and then safely inserted into a table. To enable the asynchronous mode, switch on the [async_insert](../../operations/settings/settings.md#async-insert) setting. Note that asynchronous insertions are supported only over HTTP protocol, and deduplication is not supported for them. +It's also possible to asynchronously insert data in small but frequent inserts. The data from such insertions is combined into batches and then safely inserted into a table. To enable the asynchronous mode, switch on the [async_insert](../../operations/settings/settings.md#async-insert) setting. Note that asynchronous insertions are supported only over HTTP protocol, and deduplication is not supported for them. **See Also** diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index fc0b2dd7fc3..6d5bcda8452 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -3643,7 +3643,7 @@ SELECT * FROM positional_arguments ORDER BY 2,3; ## async_insert {#async-insert} -Включает или отключает асинхронные вставки для нескольких параллельных соединений. Работает только для вставок по протоколу HTTP. Обратите внимание, что при таких вставках дедупликация не производится. +Включает или отключает асинхронные вставки. Работает только для вставок по протоколу HTTP. Обратите внимание, что при таких вставках дедупликация не производится. Если включено, данные собираются в пачки перед вставкой в таблицу. Это позволяет производить мелкие и частые вставки в ClickHouse (до 15000 запросов в секунду) без промежуточных таблиц. diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index 7fc801b3b88..baa4d83626d 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -121,7 +121,7 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - Данные поступают в режиме реального времени. - Вы загружаете данные, которые как правило отсортированы по времени. -Также возможно вставлять данные асинхронно во множественных маленьких вставках от нескольких параллельных соединений. Данные от таких вставок сначала собираются в пачки, а потом вставляются в таблицу. Чтобы включить асинхронный режим, используйте настройку [async_insert](../../operations/settings/settings.md#async-insert). Обратите внимание, что асинхронные вставки поддерживаются только через протокол HTTP, а дедупликация при этом не производится. +Также возможно вставлять данные асинхронно во множественных маленьких вставках. Данные от таких вставок сначала собираются в пачки, а потом вставляются в таблицу. Чтобы включить асинхронный режим, используйте настройку [async_insert](../../operations/settings/settings.md#async-insert). Обратите внимание, что асинхронные вставки поддерживаются только через протокол HTTP, а дедупликация при этом не производится. **См. также** From ac4a9bcf23f36e76d145de72c84cb5c8570a6094 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 27 Oct 2021 18:19:41 +0300 Subject: [PATCH 159/173] Fix 00975_move_partition_merge_tree (#30717) * Revert "Fix race between MOVE PARTITION and merges/mutations for MergeTree" This reverts commit e9ce859b022751d7511da4730432db0f592eb96d. * fix * Update StorageMergeTree.cpp Co-authored-by: alesapin --- src/Storages/StorageMergeTree.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 84022e3278d..7322d57fed2 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1179,6 +1179,7 @@ bool StorageMergeTree::optimize( ActionLock StorageMergeTree::stopMergesAndWait() { + /// TODO allow to stop merges in specific partition only (like it's done in ReplicatedMergeTree) std::unique_lock lock(currently_processing_in_background_mutex); /// Asks to complete merges and does not allow them to start. @@ -1332,6 +1333,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con { auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); auto lock2 = source_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto merges_blocker = stopMergesAndWait(); auto source_metadata_snapshot = source_table->getInMemoryMetadataPtr(); auto my_metadata_snapshot = getInMemoryMetadataPtr(); @@ -1403,13 +1405,9 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr local_context) { - /// MOVE PARTITION cannot be run in parallel with merges/mutations, - /// since otherwise there can be some merge/mutation in progress, - /// that will be created in the source table after MOVE PARTITION. - std::unique_lock background_lock(currently_processing_in_background_mutex); - auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); auto lock2 = dest_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto merges_blocker = stopMergesAndWait(); auto dest_table_storage = std::dynamic_pointer_cast(dest_table); if (!dest_table_storage) From d6c0cde173d2e682aa2f6721831298330d2d6c0d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 27 Oct 2021 18:49:18 +0300 Subject: [PATCH 160/173] Updated UserDefinedSQLFunctionFactory --- src/Common/ErrorCodes.cpp | 2 +- .../InterpreterCreateFunctionQuery.cpp | 42 ++++------- .../InterpreterDropFunctionQuery.cpp | 20 +----- .../UserDefinedExecutableFunctionFactory.cpp | 9 +++ .../UserDefinedExecutableFunctionFactory.h | 2 + .../UserDefinedSQLFunctionFactory.cpp | 69 +++++++++++++++++-- .../UserDefinedSQLFunctionFactory.h | 16 +++-- .../test.py | 5 +- 8 files changed, 105 insertions(+), 60 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 2aadda426e7..3cd914a89bb 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -577,7 +577,7 @@ M(607, BACKUP_ELEMENT_DUPLICATE) \ M(608, CANNOT_RESTORE_TABLE) \ M(609, FUNCTION_ALREADY_EXISTS) \ - M(610, CANNOT_DROP_SYSTEM_FUNCTION) \ + M(610, CANNOT_DROP_FUNCTION) \ M(611, CANNOT_CREATE_RECURSIVE_FUNCTION) \ M(612, OBJECT_ALREADY_STORED_ON_DISK) \ M(613, OBJECT_WAS_NOT_STORED_ON_DISK) \ diff --git a/src/Interpreters/InterpreterCreateFunctionQuery.cpp b/src/Interpreters/InterpreterCreateFunctionQuery.cpp index a995eed114f..d39e22f55f0 100644 --- a/src/Interpreters/InterpreterCreateFunctionQuery.cpp +++ b/src/Interpreters/InterpreterCreateFunctionQuery.cpp @@ -24,15 +24,15 @@ namespace ErrorCodes BlockIO InterpreterCreateFunctionQuery::execute() { FunctionNameNormalizer().visit(query_ptr.get()); - auto * create_function_query = query_ptr->as(); - - if (!create_function_query) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Expected CREATE FUNCTION query"); + ASTCreateFunctionQuery & create_function_query = query_ptr->as(); AccessRightsElements access_rights_elements; access_rights_elements.emplace_back(AccessType::CREATE_FUNCTION); - if (!create_function_query->cluster.empty()) + if (create_function_query.or_replace) + access_rights_elements.emplace_back(AccessType::DROP_FUNCTION); + + if (!create_function_query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, getContext(), access_rights_elements); auto current_context = getContext(); @@ -40,34 +40,16 @@ BlockIO InterpreterCreateFunctionQuery::execute() auto & user_defined_function_factory = UserDefinedSQLFunctionFactory::instance(); - auto & function_name = create_function_query->function_name; + auto & function_name = create_function_query.function_name; - bool if_not_exists = create_function_query->if_not_exists; - bool replace = create_function_query->or_replace; + bool if_not_exists = create_function_query.if_not_exists; + bool replace = create_function_query.or_replace; - create_function_query->if_not_exists = false; - create_function_query->or_replace = false; + create_function_query.if_not_exists = false; + create_function_query.or_replace = false; - if (if_not_exists && user_defined_function_factory.tryGet(function_name) != nullptr) - return {}; - - validateFunction(create_function_query->function_core, function_name); - - user_defined_function_factory.registerFunction(function_name, query_ptr, replace); - - if (persist_function) - { - try - { - UserDefinedSQLObjectsLoader::instance().storeObject(current_context, UserDefinedSQLObjectType::Function, function_name, *query_ptr, replace); - } - catch (Exception & exception) - { - user_defined_function_factory.unregisterFunction(function_name); - exception.addMessage(fmt::format("while storing user defined function {} on disk", backQuote(function_name))); - throw; - } - } + validateFunction(create_function_query.function_core, function_name); + user_defined_function_factory.registerFunction(current_context, function_name, query_ptr, replace, if_not_exists, persist_function); return {}; } diff --git a/src/Interpreters/InterpreterDropFunctionQuery.cpp b/src/Interpreters/InterpreterDropFunctionQuery.cpp index 5cc32d30f24..be67ea68cdd 100644 --- a/src/Interpreters/InterpreterDropFunctionQuery.cpp +++ b/src/Interpreters/InterpreterDropFunctionQuery.cpp @@ -12,35 +12,21 @@ namespace DB { -namespace ErrorCodes -{ - extern const int UNSUPPORTED_METHOD; -} - BlockIO InterpreterDropFunctionQuery::execute() { FunctionNameNormalizer().visit(query_ptr.get()); - auto * drop_function_query = query_ptr->as(); - - if (!drop_function_query) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Expected DROP FUNCTION query"); + ASTDropFunctionQuery & drop_function_query = query_ptr->as(); AccessRightsElements access_rights_elements; access_rights_elements.emplace_back(AccessType::DROP_FUNCTION); - if (!drop_function_query->cluster.empty()) + if (!drop_function_query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, getContext(), access_rights_elements); auto current_context = getContext(); current_context->checkAccess(access_rights_elements); - auto & user_defined_functions_factory = UserDefinedSQLFunctionFactory::instance(); - - if (drop_function_query->if_exists && !user_defined_functions_factory.has(drop_function_query->function_name)) - return {}; - - UserDefinedSQLFunctionFactory::instance().unregisterFunction(drop_function_query->function_name); - UserDefinedSQLObjectsLoader::instance().removeObject(current_context, UserDefinedSQLObjectType::Function, drop_function_query->function_name); + UserDefinedSQLFunctionFactory::instance().unregisterFunction(current_context, drop_function_query.function_name, drop_function_query.if_exists); return {}; } diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp index cfa1171a84b..4cb3e034b01 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.cpp @@ -206,6 +206,15 @@ FunctionOverloadResolverPtr UserDefinedExecutableFunctionFactory::tryGet(const S return nullptr; } +bool UserDefinedExecutableFunctionFactory::has(const String & function_name, ContextPtr context) +{ + const auto & loader = context->getExternalUserDefinedExecutableFunctionsLoader(); + auto load_result = loader.getLoadResult(function_name); + + bool result = load_result.object != nullptr; + return result; +} + std::vector UserDefinedExecutableFunctionFactory::getRegisteredNames(ContextPtr context) { const auto & loader = context->getExternalUserDefinedExecutableFunctionsLoader(); diff --git a/src/Interpreters/UserDefinedExecutableFunctionFactory.h b/src/Interpreters/UserDefinedExecutableFunctionFactory.h index 0afeeecbca7..989db4c481b 100644 --- a/src/Interpreters/UserDefinedExecutableFunctionFactory.h +++ b/src/Interpreters/UserDefinedExecutableFunctionFactory.h @@ -24,6 +24,8 @@ public: static FunctionOverloadResolverPtr tryGet(const String & function_name, ContextPtr context); + static bool has(const String & function_name, ContextPtr context); + static std::vector getRegisteredNames(ContextPtr context); }; diff --git a/src/Interpreters/UserDefinedSQLFunctionFactory.cpp b/src/Interpreters/UserDefinedSQLFunctionFactory.cpp index f036741ca21..afe51de99a4 100644 --- a/src/Interpreters/UserDefinedSQLFunctionFactory.cpp +++ b/src/Interpreters/UserDefinedSQLFunctionFactory.cpp @@ -1,7 +1,13 @@ #include "UserDefinedSQLFunctionFactory.h" +#include + #include #include +#include +#include +#include + namespace DB { @@ -10,7 +16,7 @@ namespace ErrorCodes { extern const int FUNCTION_ALREADY_EXISTS; extern const int UNKNOWN_FUNCTION; - extern const int CANNOT_DROP_SYSTEM_FUNCTION; + extern const int CANNOT_DROP_FUNCTION; } UserDefinedSQLFunctionFactory & UserDefinedSQLFunctionFactory::instance() @@ -19,13 +25,31 @@ UserDefinedSQLFunctionFactory & UserDefinedSQLFunctionFactory::instance() return result; } -void UserDefinedSQLFunctionFactory::registerFunction(const String & function_name, ASTPtr create_function_query, bool replace) +void UserDefinedSQLFunctionFactory::registerFunction(ContextPtr context, const String & function_name, ASTPtr create_function_query, bool replace, bool if_not_exists, bool persist) { if (FunctionFactory::instance().hasNameOrAlias(function_name)) + { + if (if_not_exists) + return; + throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The function '{}' already exists", function_name); + } if (AggregateFunctionFactory::instance().hasNameOrAlias(function_name)) + { + if (if_not_exists) + return; + throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The aggregate function '{}' already exists", function_name); + } + + if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) + { + if (if_not_exists) + return; + + throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "User defined executable function '{}'", function_name); + } std::lock_guard lock(mutex); @@ -33,28 +57,63 @@ void UserDefinedSQLFunctionFactory::registerFunction(const String & function_nam if (!inserted) { + if (if_not_exists) + return; + if (replace) - it->second = std::move(create_function_query); + it->second = create_function_query; else throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The function name '{}' is not unique", function_name); } + + if (persist) + { + try + { + UserDefinedSQLObjectsLoader::instance().storeObject(context, UserDefinedSQLObjectType::Function, function_name, *create_function_query, replace); + } + catch (Exception & exception) + { + function_name_to_create_query.erase(it); + exception.addMessage(fmt::format("while storing user defined function {} on disk", backQuote(function_name))); + throw; + } + } } -void UserDefinedSQLFunctionFactory::unregisterFunction(const String & function_name) +void UserDefinedSQLFunctionFactory::unregisterFunction(ContextPtr context, const String & function_name, bool if_exists) { if (FunctionFactory::instance().hasNameOrAlias(function_name) || AggregateFunctionFactory::instance().hasNameOrAlias(function_name)) - throw Exception(ErrorCodes::CANNOT_DROP_SYSTEM_FUNCTION, "Cannot drop system function '{}'", function_name); + throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "Cannot drop system function '{}'", function_name); + + if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) + throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "Cannot drop user defined executable function '{}'", function_name); std::lock_guard lock(mutex); auto it = function_name_to_create_query.find(function_name); if (it == function_name_to_create_query.end()) + { + if (if_exists) + return; + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "The function name '{}' is not registered", function_name); + } + + try + { + UserDefinedSQLObjectsLoader::instance().removeObject(context, UserDefinedSQLObjectType::Function, function_name); + } + catch (Exception & exception) + { + exception.addMessage(fmt::format("while removing user defined function {} from disk", backQuote(function_name))); + throw; + } function_name_to_create_query.erase(it); } diff --git a/src/Interpreters/UserDefinedSQLFunctionFactory.h b/src/Interpreters/UserDefinedSQLFunctionFactory.h index 6487b951705..63bf5d73c15 100644 --- a/src/Interpreters/UserDefinedSQLFunctionFactory.h +++ b/src/Interpreters/UserDefinedSQLFunctionFactory.h @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -17,13 +19,17 @@ public: static UserDefinedSQLFunctionFactory & instance(); /** Register function for function_name in factory for specified create_function_query. - * If replace = true and function with function_name already exists replace it with create_function_query. - * Otherwise throws exception. + * If function exists and if_not_exists = false and replace = false throws exception. + * If replace = true and sql user defined function with function_name already exists replace it with create_function_query. + * If persist = true persist function on disk. */ - void registerFunction(const String & function_name, ASTPtr create_function_query, bool replace); + void registerFunction(ContextPtr context, const String & function_name, ASTPtr create_function_query, bool replace, bool if_not_exists, bool persist); - /// Unregister function for function_name - void unregisterFunction(const String & function_name); + /** Unregister function for function_name. + * If if_exists = true then do not throw exception if function is not registered. + * If if_exists = false then throw exception if function is not registered. + */ + void unregisterFunction(ContextPtr context, const String & function_name, bool if_exists); /// Get function create query for function_name. If no function registered with function_name throws exception. ASTPtr get(const String & function_name) const; diff --git a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py index 48f5733b9e4..d5c74a99622 100644 --- a/tests/integration/test_sql_user_defined_functions_on_cluster/test.py +++ b/tests/integration/test_sql_user_defined_functions_on_cluster/test.py @@ -22,12 +22,13 @@ def test_sql_user_defined_functions_on_cluster(): assert "Unknown function test_function" in ch2.query_and_get_error("SELECT test_function(1);") assert "Unknown function test_function" in ch3.query_and_get_error("SELECT test_function(1);") - ch1.query_with_retry("CREATE FUNCTION test_function ON CLUSTER 'cluster' AS x -> x + 1;", retry_count=5) + ch1.query_with_retry("CREATE FUNCTION test_function ON CLUSTER 'cluster' AS x -> x + 1;") + assert ch1.query("SELECT test_function(1);") == "2\n" assert ch2.query("SELECT test_function(1);") == "2\n" assert ch3.query("SELECT test_function(1);") == "2\n" - ch2.query_with_retry("DROP FUNCTION test_function ON CLUSTER 'cluster'", retry_count=5) + ch2.query_with_retry("DROP FUNCTION test_function ON CLUSTER 'cluster'") assert "Unknown function test_function" in ch1.query_and_get_error("SELECT test_function(1);") assert "Unknown function test_function" in ch2.query_and_get_error("SELECT test_function(1);") assert "Unknown function test_function" in ch3.query_and_get_error("SELECT test_function(1);") From 18dc17f9c74805e916fe7bcd263ebbb6d0b48334 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Oct 2021 18:52:52 +0300 Subject: [PATCH 161/173] Update 01083_expressions_in_engine_arguments.sql --- .../0_stateless/01083_expressions_in_engine_arguments.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 9a29117ff5d..dd39277ee31 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -83,7 +83,7 @@ INSERT INTO buffer VALUES (1); SELECT sum(n) from rich_syntax; -- Clear cache to avoid future errors in the logs -SYSTEM DROP DNS CACHE +SYSTEM DROP DNS CACHE; DROP TABLE file; DROP TABLE url; From afc0b2068c2d53baf100565de778f51845f59bf2 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 28 Oct 2021 01:04:16 +0300 Subject: [PATCH 162/173] Update Settings.h --- src/Core/Settings.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 1dff3e1988f..b4c5f8fff59 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -264,7 +264,6 @@ class IColumn; M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ - M(UInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \ M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ From 8b4a6a24161c701ff7255bc14f1e39b5f896049d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 Oct 2021 02:10:39 +0300 Subject: [PATCH 163/173] Remove cruft --- base/base/DateLUTImpl.cpp | 4 --- base/base/LineReader.cpp | 2 -- base/base/defines.h | 4 --- base/base/logger_useful.h | 9 ------ base/base/sort.h | 21 ++------------ base/daemon/BaseDaemon.cpp | 4 +-- base/daemon/SentryWriter.cpp | 8 ++---- base/mysqlxx/Query.cpp | 2 +- src/Access/GSSAcceptor.h | 4 +-- src/Access/LDAPClient.h | 4 +-- src/AggregateFunctions/AggregateFunctionAvg.h | 4 +-- .../AggregateFunctionBitwise.h | 4 +-- .../AggregateFunctionCount.h | 4 +-- src/AggregateFunctions/AggregateFunctionIf.h | 4 +-- .../AggregateFunctionMinMaxAny.h | 4 +-- .../AggregateFunctionNull.h | 4 +-- src/AggregateFunctions/AggregateFunctionSum.h | 4 +-- .../AggregateFunctionSumMap.h | 9 ++---- src/AggregateFunctions/IAggregateFunction.h | 4 +-- src/AggregateFunctions/ThetaSketchData.h | 8 ++---- .../registerAggregateFunctions.cpp | 2 -- src/Bridge/XDBCBridgeHelper.h | 4 +-- src/Client/ClientBase.cpp | 4 +-- src/Client/Connection.cpp | 6 ++-- src/Client/Connection.h | 4 +-- src/Client/ConnectionParameters.cpp | 6 +--- src/Columns/Collator.cpp | 4 +-- src/Common/ClickHouseRevision.cpp | 5 +--- src/Common/CompactArray.h | 22 --------------- src/Common/Config/ConfigProcessor.cpp | 4 +-- src/Common/Config/ConfigProcessor.h | 4 +-- src/Common/Config/YAMLParser.cpp | 6 ++-- src/Common/Config/YAMLParser.h | 4 +-- src/Common/DNSResolver.cpp | 4 --- src/Common/Exception.cpp | 4 +-- src/Common/OpenSSLHelpers.cpp | 4 +-- src/Common/OpenSSLHelpers.h | 4 +-- src/Common/OptimizedRegularExpression.h | 4 +-- src/Common/QueryProfiler.h | 4 +-- src/Common/SparseHashMap.h | 22 +++------------ src/Common/StackTrace.cpp | 6 ++-- src/Common/ThreadStatus.cpp | 6 ++-- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 4 +-- src/Compression/CompressionCodecEncrypted.cpp | 8 ++---- src/Compression/CompressionCodecEncrypted.h | 3 -- src/Compression/CompressionFactory.cpp | 4 +-- src/Coordination/Changelog.h | 2 +- src/Coordination/InMemoryLogStore.h | 2 +- src/Coordination/KeeperDispatcher.h | 6 ++-- src/Coordination/KeeperLogStore.h | 2 +- src/Coordination/KeeperServer.cpp | 4 +-- src/Coordination/KeeperServer.h | 2 +- src/Coordination/KeeperSnapshotManager.h | 2 +- src/Coordination/KeeperStateMachine.h | 2 +- src/Coordination/KeeperStateManager.h | 2 +- src/Coordination/LoggerWrapper.h | 2 +- src/Coordination/ReadBufferFromNuraftBuffer.h | 2 +- src/Coordination/SummingStateMachine.h | 2 +- .../WriteBufferFromNuraftBuffer.h | 2 +- src/Coordination/tests/gtest_coordination.cpp | 8 ++---- src/Core/MySQL/Authentication.h | 4 +-- src/Core/PostgreSQL/Connection.h | 4 +-- src/Core/PostgreSQL/ConnectionHolder.h | 4 +-- src/Core/PostgreSQL/PoolWithFailover.h | 2 -- src/Core/PostgreSQL/Utils.h | 4 +-- src/Core/PostgreSQL/insertPostgreSQLValue.cpp | 2 +- src/Core/PostgreSQL/insertPostgreSQLValue.h | 2 -- src/DataTypes/Native.h | 4 +-- src/Databases/DatabaseFactory.cpp | 6 ++-- .../MySQL/DatabaseMaterializedMySQL.cpp | 4 +-- src/Databases/MySQL/DatabaseMySQL.cpp | 4 +-- .../MySQL/FetchTablesColumnsList.cpp | 4 +-- src/Databases/MySQL/MaterializeMetadata.h | 4 +-- .../MySQL/MaterializedMySQLSyncThread.cpp | 4 +-- .../MySQL/MaterializedMySQLSyncThread.h | 4 +-- .../DatabaseMaterializedPostgreSQL.h | 2 -- src/Databases/PostgreSQL/DatabasePostgreSQL.h | 2 -- .../fetchPostgreSQLTableStructure.h | 2 -- src/Databases/SQLite/DatabaseSQLite.h | 4 +-- src/Databases/SQLite/SQLiteUtils.h | 4 +-- .../SQLite/fetchSQLiteTableStructure.h | 4 +-- src/Dictionaries/CassandraHelpers.h | 4 +-- src/Dictionaries/CassandraSource.cpp | 2 -- src/Dictionaries/MySQLDictionarySource.h | 4 +-- src/Dictionaries/PostgreSQLDictionarySource.h | 2 -- src/Dictionaries/registerDictionaries.cpp | 4 --- src/Disks/DiskEncrypted.h | 2 -- src/Disks/IDiskRemote.h | 2 -- src/Disks/ReadIndirectBufferFromRemoteFS.h | 2 -- src/Disks/S3/DiskS3.cpp | 18 ++++++------ src/Disks/S3/DiskS3.h | 2 -- src/Disks/S3/ProxyConfiguration.h | 4 +-- src/Disks/S3/ProxyListConfiguration.h | 2 -- src/Disks/S3/ProxyResolverConfiguration.h | 2 -- src/Disks/S3/registerDiskS3.cpp | 6 ++-- src/Disks/WriteIndirectBufferFromRemoteFS.h | 2 -- src/Disks/registerDisks.cpp | 4 +-- src/Formats/ProtobufReader.h | 4 +-- src/Formats/ProtobufSchemas.cpp | 4 +-- src/Formats/ProtobufSerializer.h | 4 +-- src/Formats/ProtobufWriter.h | 4 +-- src/Formats/registerFormats.cpp | 8 +----- src/Functions/DivisionUtils.h | 4 +-- src/Functions/FunctionBase64Conversion.h | 4 +-- src/Functions/FunctionBinaryArithmetic.h | 4 +-- src/Functions/FunctionIfBase.h | 4 +-- src/Functions/FunctionMathBinaryFloat64.h | 4 +-- src/Functions/FunctionMathUnary.h | 4 +-- src/Functions/FunctionSQLJSON.h | 4 +-- src/Functions/FunctionUnaryArithmetic.h | 4 +-- src/Functions/FunctionsAES.h | 4 +-- src/Functions/FunctionsEmbeddedDictionaries.h | 4 +-- src/Functions/FunctionsHashing.cpp | 2 -- src/Functions/FunctionsHashing.h | 10 ++----- src/Functions/FunctionsJSON.cpp | 4 +-- src/Functions/GCDLCMImpl.h | 4 +-- src/Functions/IFunction.cpp | 4 +-- src/Functions/IFunction.h | 4 +-- src/Functions/MatchImpl.h | 6 ++-- src/Functions/MultiMatchAllIndicesImpl.h | 6 ++-- src/Functions/MultiMatchAnyImpl.h | 6 ++-- src/Functions/RapidJSONParser.h | 4 +-- src/Functions/Regexps.h | 4 +-- src/Functions/ReplaceRegexpImpl.h | 6 ++-- src/Functions/SimdJSONParser.h | 4 +-- src/Functions/aes_decrypt_mysql.cpp | 4 +-- src/Functions/aes_encrypt_mysql.cpp | 4 +-- src/Functions/base64Encode.cpp | 4 +-- src/Functions/convertCharset.cpp | 4 +-- src/Functions/decrypt.cpp | 4 +-- src/Functions/divide/divide.cpp | 4 +-- src/Functions/divide/divideImpl.cpp | 8 ------ src/Functions/encrypt.cpp | 4 +-- src/Functions/geoToH3.cpp | 4 +-- src/Functions/geoToS2.cpp | 4 +-- src/Functions/h3EdgeAngle.cpp | 4 +-- src/Functions/h3EdgeLengthM.cpp | 4 +-- src/Functions/h3GetBaseCell.cpp | 4 +-- src/Functions/h3GetFaces.cpp | 4 +-- src/Functions/h3GetResolution.cpp | 4 +-- src/Functions/h3HexAreaM2.cpp | 4 +-- src/Functions/h3IndexesAreNeighbors.cpp | 4 +-- src/Functions/h3IsPentagon.cpp | 4 +-- src/Functions/h3IsResClassIII.cpp | 4 +-- src/Functions/h3IsValid.cpp | 4 +-- src/Functions/h3ToChildren.cpp | 4 +-- src/Functions/h3ToGeoBoundary.cpp | 4 +-- src/Functions/h3ToParent.cpp | 4 +-- src/Functions/h3ToString.cpp | 4 +-- src/Functions/h3kRing.cpp | 4 +-- src/Functions/h3toGeo.cpp | 4 +-- src/Functions/lemmatize.cpp | 4 +-- src/Functions/normalizeString.cpp | 4 +-- src/Functions/registerFunctions.cpp | 6 +--- src/Functions/registerFunctionsGeo.cpp | 4 +-- .../registerFunctionsMiscellaneous.cpp | 4 +-- src/Functions/registerFunctionsString.cpp | 6 ++-- src/Functions/s2CapContains.cpp | 4 +-- src/Functions/s2CapUnion.cpp | 4 +-- src/Functions/s2CellsIntersect.cpp | 4 +-- src/Functions/s2GetNeighbors.cpp | 4 +-- src/Functions/s2RectAdd.cpp | 4 +-- src/Functions/s2RectContains.cpp | 4 +-- src/Functions/s2RectIntersection.cpp | 4 +-- src/Functions/s2RectUnion.cpp | 4 +-- src/Functions/s2ToGeo.cpp | 4 +-- src/Functions/s2_fwd.h | 12 ++++---- src/Functions/serverConstants.cpp | 4 +-- src/Functions/stem.cpp | 6 ++-- src/Functions/stringToH3.cpp | 4 +-- src/Functions/synonyms.cpp | 4 +-- src/IO/BrotliReadBuffer.cpp | 4 +-- src/IO/BrotliWriteBuffer.cpp | 4 +-- src/IO/Bzip2ReadBuffer.cpp | 6 ++-- src/IO/Bzip2WriteBuffer.cpp | 6 ++-- src/IO/CompressionMethod.cpp | 4 +-- src/IO/FileEncryptionCommon.h | 2 -- src/IO/HTTPCommon.cpp | 6 ++-- src/IO/LZMADeflatingWriteBuffer.cpp | 3 -- src/IO/LZMADeflatingWriteBuffer.h | 28 +------------------ src/IO/LZMAInflatingReadBuffer.cpp | 2 -- src/IO/LZMAInflatingReadBuffer.h | 26 +---------------- src/IO/ReadBufferFromEncryptedFile.h | 2 -- src/IO/ReadBufferFromS3.h | 2 -- src/IO/ReadWriteBufferFromHTTP.h | 4 +-- src/IO/S3/PocoHTTPClient.h | 10 +++---- src/IO/S3Common.h | 6 ++-- src/IO/UseSSL.cpp | 4 +-- src/IO/WriteBufferFromEncryptedFile.h | 2 -- src/IO/WriteBufferFromS3.h | 4 +-- src/IO/tests/gtest_file_encryption.cpp | 2 -- src/Interpreters/ActionsDAG.h | 4 +-- src/Interpreters/AsynchronousMetrics.cpp | 4 +-- src/Interpreters/AsynchronousMetrics.h | 15 ---------- src/Interpreters/ClientInfo.cpp | 4 +-- src/Interpreters/ClientInfo.h | 5 ---- src/Interpreters/Context.cpp | 6 ---- src/Interpreters/Context.h | 4 +-- src/Interpreters/CrashLog.cpp | 4 +-- src/Interpreters/DatabaseCatalog.cpp | 4 +-- src/Interpreters/ExpressionActions.h | 4 +-- src/Interpreters/ExpressionJIT.cpp | 4 +-- .../ExternalDictionariesLoader.cpp | 4 +-- src/Interpreters/InterpreterDropQuery.cpp | 4 +-- .../InterpreterExternalDDLQuery.cpp | 4 +-- src/Interpreters/InterpreterSystemQuery.cpp | 4 +-- src/Interpreters/JIT/CHJIT.h | 10 +++---- src/Interpreters/JIT/CompileDAG.h | 4 +-- .../JIT/CompiledExpressionCache.h | 4 +-- src/Interpreters/JIT/compileFunction.h | 4 +-- src/Interpreters/Lemmatizers.cpp | 9 ++---- src/Interpreters/Lemmatizers.h | 4 +-- .../MySQL/tests/gtest_create_rewritten.cpp | 4 +-- src/Interpreters/Session.cpp | 9 ------ src/Interpreters/SynonymsExtensions.cpp | 6 ++-- src/Interpreters/SynonymsExtensions.h | 4 +-- src/Interpreters/examples/jit_example.cpp | 4 +-- src/Interpreters/executeQuery.cpp | 2 -- src/Parsers/ASTSystemQuery.h | 4 +-- src/Parsers/ParserExternalDDLQuery.cpp | 4 +-- .../Formats/Impl/ArrowBlockInputFormat.h | 4 +-- .../Formats/Impl/ArrowBlockOutputFormat.h | 4 +-- .../Formats/Impl/ArrowBufferedStreams.h | 4 +-- .../Formats/Impl/ArrowColumnToCHColumn.h | 4 +-- .../Formats/Impl/CHColumnToArrowColumn.cpp | 2 -- .../Formats/Impl/CHColumnToArrowColumn.h | 5 +--- .../Formats/Impl/MsgPackRowInputFormat.h | 7 ++--- .../Formats/Impl/MsgPackRowOutputFormat.h | 6 ++-- .../Formats/Impl/ORCBlockInputFormat.h | 4 +-- .../Formats/Impl/ORCBlockOutputFormat.h | 2 -- .../Formats/Impl/ParquetBlockInputFormat.h | 4 +-- .../Formats/Impl/ParquetBlockOutputFormat.h | 4 +-- .../Formats/Impl/ProtobufRowInputFormat.h | 4 +-- .../Formats/Impl/ProtobufRowOutputFormat.h | 4 +-- src/Processors/Sources/MySQLSource.cpp | 2 -- src/Processors/Sources/SQLiteSource.h | 4 +-- src/Processors/Transforms/PostgreSQLSource.h | 2 -- src/Server/GRPCServer.h | 2 -- .../WriteBufferFromHTTPServerResponse.cpp | 4 +-- src/Server/HTTPHandler.cpp | 8 +----- src/Server/KeeperTCPHandler.h | 6 ++-- src/Server/MySQLHandler.cpp | 4 +-- src/Server/MySQLHandler.h | 4 +-- src/Server/MySQLHandlerFactory.h | 4 +-- src/Server/PostgreSQLHandler.cpp | 4 +-- src/Server/ProtocolServerAdapter.h | 2 -- src/Server/TCPHandler.cpp | 4 +-- src/Storages/HDFS/HDFSCommon.h | 4 +-- src/Storages/HDFS/ReadBufferFromHDFS.h | 6 ++-- src/Storages/HDFS/StorageHDFS.h | 2 -- src/Storages/HDFS/WriteBufferFromHDFS.h | 4 +-- .../MaterializedPostgreSQLSettings.h | 2 -- .../StorageMaterializedPostgreSQL.h | 2 -- .../ReadFinalForExternalReplicaStorage.h | 4 +-- src/Storages/StorageExternalDistributed.h | 2 -- src/Storages/StorageMaterializedMySQL.cpp | 4 +-- src/Storages/StorageMaterializedMySQL.h | 2 -- src/Storages/StorageMongoDBSocketFactory.cpp | 4 +-- src/Storages/StorageMySQL.h | 2 -- src/Storages/StoragePostgreSQL.h | 2 -- src/Storages/StorageS3Cluster.cpp | 2 -- src/Storages/StorageS3Cluster.h | 2 -- src/Storages/StorageSQLite.h | 4 +-- src/Storages/System/IStorageSystemOneBlock.h | 5 ---- .../System/StorageSystemAsynchronousMetrics.h | 6 ---- .../System/StorageSystemBuildOptions.cpp | 4 --- src/Storages/System/attachSystemTables.cpp | 15 ++-------- src/Storages/registerStorages.cpp | 6 ++-- src/TableFunctions/ITableFunctionXDBC.h | 4 +-- src/TableFunctions/TableFunctionMySQL.cpp | 2 -- src/TableFunctions/TableFunctionMySQL.h | 4 +-- src/TableFunctions/TableFunctionPostgreSQL.h | 2 -- src/TableFunctions/TableFunctionS3Cluster.cpp | 2 -- src/TableFunctions/TableFunctionS3Cluster.h | 2 -- src/TableFunctions/TableFunctionSQLite.h | 2 -- src/TableFunctions/registerTableFunctions.h | 6 ++-- 276 files changed, 278 insertions(+), 980 deletions(-) diff --git a/base/base/DateLUTImpl.cpp b/base/base/DateLUTImpl.cpp index 09ed0b1495d..bbce3b111d3 100644 --- a/base/base/DateLUTImpl.cpp +++ b/base/base/DateLUTImpl.cpp @@ -177,8 +177,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) } -#if !defined(ARCADIA_BUILD) /// Arcadia's variant of CCTZ already has the same implementation. - /// Prefer to load timezones from blobs linked to the binary. /// The blobs are provided by "tzdata" library. /// This allows to avoid dependency on system tzdata. @@ -234,5 +232,3 @@ namespace cctz_extension ZoneInfoSourceFactory zone_info_source_factory = custom_factory; } - -#endif diff --git a/base/base/LineReader.cpp b/base/base/LineReader.cpp index d325154ee61..5beebb58b3b 100644 --- a/base/base/LineReader.cpp +++ b/base/base/LineReader.cpp @@ -103,7 +103,6 @@ String LineReader::readLine(const String & first_prompt, const String & second_p continue; } -#if !defined(ARCADIA_BUILD) /// C++20 const char * has_extender = nullptr; for (const auto * extender : extenders) { @@ -133,7 +132,6 @@ String LineReader::readLine(const String & first_prompt, const String & second_p if (input.empty()) continue; } -#endif line += (line.empty() ? "" : "\n") + input; diff --git a/base/base/defines.h b/base/base/defines.h index aef901d3148..21a3c09f532 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -83,10 +83,6 @@ # define BOOST_USE_UCONTEXT 1 #endif -#if defined(ARCADIA_BUILD) && defined(BOOST_USE_UCONTEXT) -# undef BOOST_USE_UCONTEXT -#endif - /// TODO: Strange enough, there is no way to detect UB sanitizer. /// Explicitly allow undefined behaviour for certain functions. Use it as a function attribute. diff --git a/base/base/logger_useful.h b/base/base/logger_useful.h index e2290a727b4..1237c6bd47c 100644 --- a/base/base/logger_useful.h +++ b/base/base/logger_useful.h @@ -49,12 +49,3 @@ namespace #define LOG_WARNING(logger, ...) LOG_IMPL(logger, DB::LogsLevel::warning, Poco::Message::PRIO_WARNING, __VA_ARGS__) #define LOG_ERROR(logger, ...) LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_ERROR, __VA_ARGS__) #define LOG_FATAL(logger, ...) LOG_IMPL(logger, DB::LogsLevel::error, Poco::Message::PRIO_FATAL, __VA_ARGS__) - - -/// Compatibility for external projects. -#if defined(ARCADIA_BUILD) - using Poco::Logger; - using Poco::Message; - using DB::LogsLevel; - using DB::CurrentThread; -#endif diff --git a/base/base/sort.h b/base/base/sort.h index 3f5e14e1ed2..1f12cc62218 100644 --- a/base/base/sort.h +++ b/base/base/sort.h @@ -3,41 +3,24 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" -#if !defined(ARCADIA_BUILD) -# include // Y_IGNORE -#else -# include -#endif +#include template void nth_element(RandomIt first, RandomIt nth, RandomIt last) { -#if !defined(ARCADIA_BUILD) ::miniselect::floyd_rivest_select(first, nth, last); -#else - ::std::nth_element(first, nth, last); -#endif } template void partial_sort(RandomIt first, RandomIt middle, RandomIt last) { -#if !defined(ARCADIA_BUILD) ::miniselect::floyd_rivest_partial_sort(first, middle, last); -#else - ::std::partial_sort(first, middle, last); -#endif } template void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare) { -#if !defined(ARCADIA_BUILD) ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare); -#else - ::std::partial_sort(first, middle, last, compare); -#endif +} #pragma GCC diagnostic pop - -} diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 63ce529eb0b..70915a520b3 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -63,9 +63,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if defined(OS_DARWIN) # pragma GCC diagnostic ignored "-Wunused-macros" diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index ac771b9bf47..6996b63b5dd 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -15,14 +15,12 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "Common/config_version.h" -# include -#endif +#include "Common/config_version.h" +#include #if USE_SENTRY -# include // Y_IGNORE +# include # include # include diff --git a/base/mysqlxx/Query.cpp b/base/mysqlxx/Query.cpp index c0d5c20fdfd..d4514c3e988 100644 --- a/base/mysqlxx/Query.cpp +++ b/base/mysqlxx/Query.cpp @@ -2,7 +2,7 @@ #include #include #else -#include //Y_IGNORE +#include #include #endif diff --git a/src/Access/GSSAcceptor.h b/src/Access/GSSAcceptor.h index aa11507e559..4501622cd81 100644 --- a/src/Access/GSSAcceptor.h +++ b/src/Access/GSSAcceptor.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index 143047c6f95..3d6ee9c8dc9 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index fd8f9faa3f8..a9c06562e72 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -9,9 +9,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.h b/src/AggregateFunctions/AggregateFunctionBitwise.h index 90db2469828..d79192ab38a 100644 --- a/src/AggregateFunctions/AggregateFunctionBitwise.h +++ b/src/AggregateFunctions/AggregateFunctionBitwise.h @@ -9,9 +9,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionCount.h b/src/AggregateFunctions/AggregateFunctionCount.h index 0b4052c987b..022789cc96c 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.h +++ b/src/AggregateFunctions/AggregateFunctionCount.h @@ -10,9 +10,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionIf.h b/src/AggregateFunctions/AggregateFunctionIf.h index 79999437ca1..c7c5dbf441e 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.h +++ b/src/AggregateFunctions/AggregateFunctionIf.h @@ -5,9 +5,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index 189272a7058..ad3ccdcc48e 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -14,9 +14,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 7890e96ef66..503243f98b0 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -10,9 +10,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index e7f0357f644..f6a9bdda7fd 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -14,9 +14,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # include diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 3bc1ff2b61f..e573cf08c50 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -395,9 +395,7 @@ private: using Self = AggregateFunctionSumMapFiltered; using Base = AggregateFunctionMapBase; - /// ARCADIA_BUILD disallow unordered_set for big ints for some reason - static constexpr const bool allow_hash = !is_over_big_int; - using ContainerT = std::conditional_t, std::set>; + using ContainerT = std::unordered_set; ContainerT keys_to_keep; @@ -418,13 +416,10 @@ public: "Aggregate function {} requires an Array as a parameter", getName()); - if constexpr (allow_hash) - keys_to_keep.reserve(keys_to_keep_.size()); + keys_to_keep.reserve(keys_to_keep_.size()); for (const Field & f : keys_to_keep_) - { keys_to_keep.emplace(f.safeGet()); - } } String getName() const override diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 61e4e424652..e86fc13ccc4 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -9,9 +9,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/AggregateFunctions/ThetaSketchData.h b/src/AggregateFunctions/ThetaSketchData.h index b2e32db6c75..bc583250002 100644 --- a/src/AggregateFunctions/ThetaSketchData.h +++ b/src/AggregateFunctions/ThetaSketchData.h @@ -1,15 +1,13 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_DATASKETCHES #include #include -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include namespace DB diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index 5d0af719290..bffc344062e 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -98,9 +98,7 @@ void registerAggregateFunctions() registerAggregateFunctionUniqUpTo(factory); registerAggregateFunctionTopK(factory); registerAggregateFunctionsBitwise(factory); -#if !defined(ARCADIA_BUILD) registerAggregateFunctionsBitmap(factory); -#endif registerAggregateFunctionsMaxIntersections(factory); registerAggregateFunctionHistogram(factory); registerAggregateFunctionRetention(factory); diff --git a/src/Bridge/XDBCBridgeHelper.h b/src/Bridge/XDBCBridgeHelper.h index d6d2af3a045..80d9610f837 100644 --- a/src/Bridge/XDBCBridgeHelper.h +++ b/src/Bridge/XDBCBridgeHelper.h @@ -15,9 +15,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 8a1900897c5..ed90c777e20 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -14,9 +14,7 @@ #include "Core/Block.h" #include "Core/Protocol.h" -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include #include #include diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 3c920e8cabe..d7d80606804 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -31,10 +31,8 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -# include -#endif +#include +#include #if USE_SSL # include diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 17d6b41a2e5..3b49760ba10 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -4,9 +4,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include #include diff --git a/src/Client/ConnectionParameters.cpp b/src/Client/ConnectionParameters.cpp index 9ec4d663eef..dbd463583f5 100644 --- a/src/Client/ConnectionParameters.cpp +++ b/src/Client/ConnectionParameters.cpp @@ -12,9 +12,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -#include // Y_IGNORE -#endif +#include namespace DB @@ -55,12 +53,10 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati } if (password_prompt) { -#if !defined(ARCADIA_BUILD) std::string prompt{"Password for user (" + user + "): "}; char buf[1000] = {}; if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0)) password = result; -#endif } /// By default compression is disabled if address looks like localhost. diff --git a/src/Columns/Collator.cpp b/src/Columns/Collator.cpp index d33f65e3836..312216054f5 100644 --- a/src/Columns/Collator.cpp +++ b/src/Columns/Collator.cpp @@ -1,8 +1,6 @@ #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_ICU # include diff --git a/src/Common/ClickHouseRevision.cpp b/src/Common/ClickHouseRevision.cpp index 2c52ebb064a..c7c27436466 100644 --- a/src/Common/ClickHouseRevision.cpp +++ b/src/Common/ClickHouseRevision.cpp @@ -1,8 +1,5 @@ #include - -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace ClickHouseRevision { diff --git a/src/Common/CompactArray.h b/src/Common/CompactArray.h index 7291fe3f18d..89e2b43a943 100644 --- a/src/Common/CompactArray.h +++ b/src/Common/CompactArray.h @@ -55,28 +55,6 @@ public: return locus; } - /// Used only in arcadia/metrika - void readText(ReadBuffer & in) - { - for (size_t i = 0; i < BITSET_SIZE; ++i) - { - if (i != 0) - assertChar(',', in); - readIntText(bitset[i], in); - } - } - - /// Used only in arcadia/metrika - void writeText(WriteBuffer & out) const - { - for (size_t i = 0; i < BITSET_SIZE; ++i) - { - if (i != 0) - writeCString(",", out); - writeIntText(bitset[i], out); - } - } - private: /// number of bytes in bitset static constexpr size_t BITSET_SIZE = (static_cast(bucket_count) * content_width + 7) / 8; diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 09397e31787..da7405b993f 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #include "ConfigProcessor.h" #include "YAMLParser.h" diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index caffad5faca..2a92a709934 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #include #include diff --git a/src/Common/Config/YAMLParser.cpp b/src/Common/Config/YAMLParser.cpp index 9f52d9cf36c..1d6d7166669 100644 --- a/src/Common/Config/YAMLParser.cpp +++ b/src/Common/Config/YAMLParser.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #if USE_YAML_CPP #include "YAMLParser.h" @@ -19,7 +17,7 @@ #include #include -#include // Y_IGNORE +#include #include diff --git a/src/Common/Config/YAMLParser.h b/src/Common/Config/YAMLParser.h index b983e85a4fe..b181669f45f 100644 --- a/src/Common/Config/YAMLParser.h +++ b/src/Common/Config/YAMLParser.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #include diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index d69291b79c0..79ab7f82fe0 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -114,11 +114,7 @@ static DNSResolver::IPAddresses resolveIPAddressImpl(const std::string & host) try { -#if defined(ARCADIA_BUILD) - addresses = Poco::Net::DNS::hostByName(host, &Poco::Net::DNS::DEFAULT_DNS_TIMEOUT, flags).addresses(); -#else addresses = Poco::Net::DNS::hostByName(host, flags).addresses(); -#endif } catch (const Poco::Net::DNSException & e) { diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index d506d48c25d..fc6a18fd3b9 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -17,9 +17,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace fs = std::filesystem; diff --git a/src/Common/OpenSSLHelpers.cpp b/src/Common/OpenSSLHelpers.cpp index d7e73b75f56..d73e08e79a9 100644 --- a/src/Common/OpenSSLHelpers.cpp +++ b/src/Common/OpenSSLHelpers.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL #include "OpenSSLHelpers.h" diff --git a/src/Common/OpenSSLHelpers.h b/src/Common/OpenSSLHelpers.h index e5cfcef9afe..f0dbbc10b4c 100644 --- a/src/Common/OpenSSLHelpers.h +++ b/src/Common/OpenSSLHelpers.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index fddefe596c4..f7223c2efa9 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -7,9 +7,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_RE2_ST # include diff --git a/src/Common/QueryProfiler.h b/src/Common/QueryProfiler.h index 6ef55ce871a..4014a93996f 100644 --- a/src/Common/QueryProfiler.h +++ b/src/Common/QueryProfiler.h @@ -4,9 +4,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace Poco diff --git a/src/Common/SparseHashMap.h b/src/Common/SparseHashMap.h index 0f86cc13612..3f38d52a2b8 100644 --- a/src/Common/SparseHashMap.h +++ b/src/Common/SparseHashMap.h @@ -1,24 +1,10 @@ #pragma once /// SparseHashMap is a wrapper for google::sparse_hash_map. -#if defined(ARCADIA_BUILD) -#define HASH_FUN_H -template -struct THash; -#endif #include -#if !defined(ARCADIA_BUILD) - template , - class EqualKey = std::equal_to, - class Alloc = google::libc_allocator_with_realloc>> - using SparseHashMap = google::sparse_hash_map; -#else - template , - class EqualKey = std::equal_to, - class Alloc = google::sparsehash::libc_allocator_with_realloc>> - using SparseHashMap = google::sparsehash::sparse_hash_map; - - #undef THash -#endif +template , + class EqualKey = std::equal_to, + class Alloc = google::libc_allocator_with_realloc>> +using SparseHashMap = google::sparse_hash_map; diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index fc995e435ec..6c52d31c9a1 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -13,9 +13,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_UNWIND # include @@ -201,7 +199,7 @@ void StackTrace::symbolize( const StackTrace::FramePointers & frame_pointers, [[maybe_unused]] size_t offset, size_t size, StackTrace::Frames & frames) { -#if defined(__ELF__) && !defined(__FreeBSD__) && !defined(ARCADIA_BUILD) +#if defined(__ELF__) && !defined(__FreeBSD__) auto symbol_index_ptr = DB::SymbolIndex::instance(); const DB::SymbolIndex & symbol_index = *symbol_index_ptr; diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 42bb53aed67..c976e4ca16a 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -26,7 +26,7 @@ namespace ErrorCodes thread_local ThreadStatus * current_thread = nullptr; thread_local ThreadStatus * main_thread = nullptr; -#if !defined(SANITIZER) && !defined(ARCADIA_BUILD) +#if !defined(SANITIZER) namespace { @@ -88,7 +88,7 @@ ThreadStatus::ThreadStatus() /// Will set alternative signal stack to provide diagnostics for stack overflow errors. /// If not already installed for current thread. /// Sanitizer makes larger stack usage and also it's incompatible with alternative stack by default (it sets up and relies on its own). -#if !defined(SANITIZER) && !defined(ARCADIA_BUILD) +#if !defined(SANITIZER) if (!has_alt_stack) { /// Don't repeat tries even if not installed successfully. @@ -147,11 +147,9 @@ ThreadStatus::~ThreadStatus() thread_group->threads.erase(this); } -#if !defined(ARCADIA_BUILD) /// It may cause segfault if query_context was destroyed, but was not detached auto query_context_ptr = query_context.lock(); assert((!query_context_ptr && query_id.empty()) || (query_context_ptr && query_id == query_context_ptr->getCurrentQueryId())); -#endif if (deleter) deleter(); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 63b4a61b866..09b2548f2f4 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -9,9 +9,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/Compression/CompressionCodecEncrypted.cpp b/src/Compression/CompressionCodecEncrypted.cpp index 18cd5469786..96d264fea1e 100644 --- a/src/Compression/CompressionCodecEncrypted.cpp +++ b/src/Compression/CompressionCodecEncrypted.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include #include #include @@ -13,10 +11,10 @@ // This depends on BoringSSL-specific API, notably . #if USE_SSL && USE_INTERNAL_SSL_LIBRARY #include -#include // Y_IGNORE +#include #include #include -#include // Y_IGNORE +#include #endif // Common part for both parts (with SSL and without) diff --git a/src/Compression/CompressionCodecEncrypted.h b/src/Compression/CompressionCodecEncrypted.h index 75c29ee98a1..763e4da5d56 100644 --- a/src/Compression/CompressionCodecEncrypted.h +++ b/src/Compression/CompressionCodecEncrypted.h @@ -1,6 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include #include #include @@ -134,5 +133,3 @@ private: }; } - -#endif /* NOT Arcadia_build */ diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 4dbc72c22f5..b665e898aa6 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 2fd0a927f5d..edca61a883c 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -1,6 +1,6 @@ #pragma once -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Coordination/InMemoryLogStore.h b/src/Coordination/InMemoryLogStore.h index caa5e31698d..ad4466b363f 100644 --- a/src/Coordination/InMemoryLogStore.h +++ b/src/Coordination/InMemoryLogStore.h @@ -4,7 +4,7 @@ #include #include #include -#include // Y_IGNORE +#include namespace DB { diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h index 97152ae80b8..60f69cfae9f 100644 --- a/src/Coordination/KeeperDispatcher.h +++ b/src/Coordination/KeeperDispatcher.h @@ -1,9 +1,7 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include -# include "config_core.h" -#endif +#include +#include "config_core.h" #if USE_NURAFT diff --git a/src/Coordination/KeeperLogStore.h b/src/Coordination/KeeperLogStore.h index 76343582311..be529cdcb75 100644 --- a/src/Coordination/KeeperLogStore.h +++ b/src/Coordination/KeeperLogStore.h @@ -1,5 +1,5 @@ #pragma once -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 840a0ee90bb..f4787a66d7d 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -1,9 +1,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index 7e4c5381f6e..1dc5282b285 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -1,6 +1,6 @@ #pragma once -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h index cac27f0faa8..2889ec493df 100644 --- a/src/Coordination/KeeperSnapshotManager.h +++ b/src/Coordination/KeeperSnapshotManager.h @@ -1,5 +1,5 @@ #pragma once -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index b8085c032c9..5af3dff73b5 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -2,7 +2,7 @@ #include #include -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Coordination/KeeperStateManager.h b/src/Coordination/KeeperStateManager.h index 5f051fcfc30..96f0484c05a 100644 --- a/src/Coordination/KeeperStateManager.h +++ b/src/Coordination/KeeperStateManager.h @@ -4,7 +4,7 @@ #include #include #include -#include // Y_IGNORE +#include #include #include diff --git a/src/Coordination/LoggerWrapper.h b/src/Coordination/LoggerWrapper.h index 12b24d99cc9..002fa870241 100644 --- a/src/Coordination/LoggerWrapper.h +++ b/src/Coordination/LoggerWrapper.h @@ -1,6 +1,6 @@ #pragma once -#include // Y_IGNORE +#include #include #include diff --git a/src/Coordination/ReadBufferFromNuraftBuffer.h b/src/Coordination/ReadBufferFromNuraftBuffer.h index 3817e217881..cc01d3c8f39 100644 --- a/src/Coordination/ReadBufferFromNuraftBuffer.h +++ b/src/Coordination/ReadBufferFromNuraftBuffer.h @@ -1,7 +1,7 @@ #pragma once #include -#include // Y_IGNORE +#include namespace DB { diff --git a/src/Coordination/SummingStateMachine.h b/src/Coordination/SummingStateMachine.h index 03572840739..bbb12834608 100644 --- a/src/Coordination/SummingStateMachine.h +++ b/src/Coordination/SummingStateMachine.h @@ -1,6 +1,6 @@ #pragma once -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Coordination/WriteBufferFromNuraftBuffer.h b/src/Coordination/WriteBufferFromNuraftBuffer.h index d037a0e6a27..47a01fbc2a4 100644 --- a/src/Coordination/WriteBufferFromNuraftBuffer.h +++ b/src/Coordination/WriteBufferFromNuraftBuffer.h @@ -1,7 +1,7 @@ #pragma once #include -#include // Y_IGNORE +#include namespace DB { diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 12902252328..b04af1422dc 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -1,9 +1,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include -# include "config_core.h" -#endif +#include +#include "config_core.h" #if USE_NURAFT #include @@ -22,7 +20,7 @@ #include #include #include -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h index 7a173e80e08..26194236aa2 100644 --- a/src/Core/MySQL/Authentication.h +++ b/src/Core/MySQL/Authentication.h @@ -4,9 +4,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_SSL # include diff --git a/src/Core/PostgreSQL/Connection.h b/src/Core/PostgreSQL/Connection.h index 681681a38bf..d65c38643c1 100644 --- a/src/Core/PostgreSQL/Connection.h +++ b/src/Core/PostgreSQL/Connection.h @@ -1,12 +1,10 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX -#include // Y_IGNORE +#include #include #include diff --git a/src/Core/PostgreSQL/ConnectionHolder.h b/src/Core/PostgreSQL/ConnectionHolder.h index c2a1908774c..d0d64935e91 100644 --- a/src/Core/PostgreSQL/ConnectionHolder.h +++ b/src/Core/PostgreSQL/ConnectionHolder.h @@ -1,12 +1,10 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX -#include // Y_IGNORE +#include #include #include diff --git a/src/Core/PostgreSQL/PoolWithFailover.h b/src/Core/PostgreSQL/PoolWithFailover.h index fbf2780daaf..c59010a5d43 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.h +++ b/src/Core/PostgreSQL/PoolWithFailover.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX diff --git a/src/Core/PostgreSQL/Utils.h b/src/Core/PostgreSQL/Utils.h index c80089d5741..1aa173d7137 100644 --- a/src/Core/PostgreSQL/Utils.h +++ b/src/Core/PostgreSQL/Utils.h @@ -1,12 +1,10 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX -#include // Y_IGNORE +#include #include #include "Connection.h" #include diff --git a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp index 4f31c8dfb52..1c3230ec826 100644 --- a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp +++ b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp @@ -15,7 +15,7 @@ #include #include #include -#include // Y_IGNORE +#include namespace DB diff --git a/src/Core/PostgreSQL/insertPostgreSQLValue.h b/src/Core/PostgreSQL/insertPostgreSQLValue.h index 4ed3eb95aac..dcdd9a4f9b8 100644 --- a/src/Core/PostgreSQL/insertPostgreSQLValue.h +++ b/src/Core/PostgreSQL/insertPostgreSQLValue.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 970b70f9f0b..b72e479cb1d 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER # include diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index d997977cc3a..03d91324672 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -16,9 +16,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL # include @@ -36,7 +34,7 @@ #endif #if USE_LIBPQXX -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 2b4649c275a..a2e03b10d47 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 71b0c2ec6ea..feebf33f58a 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL # include diff --git a/src/Databases/MySQL/FetchTablesColumnsList.cpp b/src/Databases/MySQL/FetchTablesColumnsList.cpp index 851c9bc74bd..529940f713c 100644 --- a/src/Databases/MySQL/FetchTablesColumnsList.cpp +++ b/src/Databases/MySQL/FetchTablesColumnsList.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL #include diff --git a/src/Databases/MySQL/MaterializeMetadata.h b/src/Databases/MySQL/MaterializeMetadata.h index 1b933fdd373..bcb0465b61e 100644 --- a/src/Databases/MySQL/MaterializeMetadata.h +++ b/src/Databases/MySQL/MaterializeMetadata.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 86481d9fd84..d0c5d12e848 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.h b/src/Databases/MySQL/MaterializedMySQLSyncThread.h index b8c985915dc..524426bada6 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.h +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h index 5770187ad09..25e32fb0312 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index c64feb74020..f801f9585d6 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h index 809378abe38..19871234e28 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX #include diff --git a/src/Databases/SQLite/DatabaseSQLite.h b/src/Databases/SQLite/DatabaseSQLite.h index 18a571dd703..d0b8d582844 100644 --- a/src/Databases/SQLite/DatabaseSQLite.h +++ b/src/Databases/SQLite/DatabaseSQLite.h @@ -1,15 +1,13 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_SQLITE #include #include #include -#include // Y_IGNORE +#include namespace DB diff --git a/src/Databases/SQLite/SQLiteUtils.h b/src/Databases/SQLite/SQLiteUtils.h index 35f00904d0d..09119b3f145 100644 --- a/src/Databases/SQLite/SQLiteUtils.h +++ b/src/Databases/SQLite/SQLiteUtils.h @@ -1,13 +1,11 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_SQLITE #include #include -#include // Y_IGNORE +#include namespace DB diff --git a/src/Databases/SQLite/fetchSQLiteTableStructure.h b/src/Databases/SQLite/fetchSQLiteTableStructure.h index 80f50173e5e..dbdf35be4bb 100644 --- a/src/Databases/SQLite/fetchSQLiteTableStructure.h +++ b/src/Databases/SQLite/fetchSQLiteTableStructure.h @@ -1,13 +1,11 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_SQLITE #include -#include // Y_IGNORE +#include namespace DB diff --git a/src/Dictionaries/CassandraHelpers.h b/src/Dictionaries/CassandraHelpers.h index 8a00e372c96..30111e11686 100644 --- a/src/Dictionaries/CassandraHelpers.h +++ b/src/Dictionaries/CassandraHelpers.h @@ -1,11 +1,9 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_CASSANDRA -#include // Y_IGNORE +#include #include #include diff --git a/src/Dictionaries/CassandraSource.cpp b/src/Dictionaries/CassandraSource.cpp index 1ebacdb2c2f..5b84936a137 100644 --- a/src/Dictionaries/CassandraSource.cpp +++ b/src/Dictionaries/CassandraSource.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_CASSANDRA diff --git a/src/Dictionaries/MySQLDictionarySource.h b/src/Dictionaries/MySQLDictionarySource.h index 1ecc41036be..37743ca2f62 100644 --- a/src/Dictionaries/MySQLDictionarySource.h +++ b/src/Dictionaries/MySQLDictionarySource.h @@ -2,9 +2,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL # include diff --git a/src/Dictionaries/PostgreSQLDictionarySource.h b/src/Dictionaries/PostgreSQLDictionarySource.h index d5ae025ea80..1cde2958107 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.h +++ b/src/Dictionaries/PostgreSQLDictionarySource.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #include "DictionaryStructure.h" #include "IDictionarySource.h" diff --git a/src/Dictionaries/registerDictionaries.cpp b/src/Dictionaries/registerDictionaries.cpp index df191edd1c3..4ade5d88bd4 100644 --- a/src/Dictionaries/registerDictionaries.cpp +++ b/src/Dictionaries/registerDictionaries.cpp @@ -14,9 +14,7 @@ void registerDictionarySourceCassandra(DictionarySourceFactory & source_factory) void registerDictionarySourceRedis(DictionarySourceFactory & source_factory); void registerDictionarySourceXDBC(DictionarySourceFactory & source_factory); void registerDictionarySourceJDBC(DictionarySourceFactory & source_factory); -#if !defined(ARCADIA_BUILD) void registerDictionarySourcePostgreSQL(DictionarySourceFactory & source_factory); -#endif void registerDictionarySourceExecutable(DictionarySourceFactory & source_factory); void registerDictionarySourceExecutablePool(DictionarySourceFactory & source_factory); void registerDictionarySourceHTTP(DictionarySourceFactory & source_factory); @@ -46,9 +44,7 @@ void registerDictionaries() registerDictionarySourceCassandra(source_factory); registerDictionarySourceXDBC(source_factory); registerDictionarySourceJDBC(source_factory); -#if !defined(ARCADIA_BUILD) registerDictionarySourcePostgreSQL(source_factory); -#endif registerDictionarySourceExecutable(source_factory); registerDictionarySourceExecutablePool(source_factory); registerDictionarySourceHTTP(source_factory); diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h index 134c0e04193..5b1bd7c5c6d 100644 --- a/src/Disks/DiskEncrypted.h +++ b/src/Disks/DiskEncrypted.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_SSL #include diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index 5333b14a12d..7453fce747e 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #include #include "Disks/DiskFactory.h" diff --git a/src/Disks/ReadIndirectBufferFromRemoteFS.h b/src/Disks/ReadIndirectBufferFromRemoteFS.h index d3cc6aeddd0..bf7f95c7987 100644 --- a/src/Disks/ReadIndirectBufferFromRemoteFS.h +++ b/src/Disks/ReadIndirectBufferFromRemoteFS.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #include #include diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index c545c436835..c8e248538c0 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -29,15 +29,15 @@ #include #include -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace DB diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index e8ce1883c38..b8f83b4763d 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/Disks/S3/ProxyConfiguration.h b/src/Disks/S3/ProxyConfiguration.h index 2e4e5bec89a..a567c2abb5a 100644 --- a/src/Disks/S3/ProxyConfiguration.h +++ b/src/Disks/S3/ProxyConfiguration.h @@ -1,14 +1,12 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 #include #include -#include // Y_IGNORE +#include #include namespace DB::S3 diff --git a/src/Disks/S3/ProxyListConfiguration.h b/src/Disks/S3/ProxyListConfiguration.h index bd5bbba19a4..c7a4ee9ec58 100644 --- a/src/Disks/S3/ProxyListConfiguration.h +++ b/src/Disks/S3/ProxyListConfiguration.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/Disks/S3/ProxyResolverConfiguration.h b/src/Disks/S3/ProxyResolverConfiguration.h index f7eba8d028a..f5baac54fba 100644 --- a/src/Disks/S3/ProxyResolverConfiguration.h +++ b/src/Disks/S3/ProxyResolverConfiguration.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 0946c8875d3..57c2cf17239 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #include #include @@ -10,7 +8,7 @@ #if USE_AWS_S3 -#include // Y_IGNORE +#include #include #include "DiskS3.h" #include "Disks/DiskCacheWrapper.h" diff --git a/src/Disks/WriteIndirectBufferFromRemoteFS.h b/src/Disks/WriteIndirectBufferFromRemoteFS.h index 69f5705356e..ef021686221 100644 --- a/src/Disks/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/WriteIndirectBufferFromRemoteFS.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #include #include diff --git a/src/Disks/registerDisks.cpp b/src/Disks/registerDisks.cpp index 94ab6098c44..5dd86043d83 100644 --- a/src/Disks/registerDisks.cpp +++ b/src/Disks/registerDisks.cpp @@ -2,9 +2,7 @@ #include "DiskFactory.h" -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB { diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h index 31d6f9a08e0..0df139eeacd 100644 --- a/src/Formats/ProtobufReader.h +++ b/src/Formats/ProtobufReader.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PROTOBUF # include diff --git a/src/Formats/ProtobufSchemas.cpp b/src/Formats/ProtobufSchemas.cpp index 9c6ed76ef27..a6e63b1c256 100644 --- a/src/Formats/ProtobufSchemas.cpp +++ b/src/Formats/ProtobufSchemas.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PROTOBUF # include diff --git a/src/Formats/ProtobufSerializer.h b/src/Formats/ProtobufSerializer.h index 86a2f2f36dd..315a138f9cf 100644 --- a/src/Formats/ProtobufSerializer.h +++ b/src/Formats/ProtobufSerializer.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PROTOBUF # include diff --git a/src/Formats/ProtobufWriter.h b/src/Formats/ProtobufWriter.h index 6af1a237fbd..c564db110cc 100644 --- a/src/Formats/ProtobufWriter.h +++ b/src/Formats/ProtobufWriter.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PROTOBUF # include diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp index acaf6f28492..a95ccb8b064 100644 --- a/src/Formats/registerFormats.cpp +++ b/src/Formats/registerFormats.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include @@ -120,10 +118,8 @@ void registerFormats() registerOutputFormatORC(factory); registerInputFormatParquet(factory); registerOutputFormatParquet(factory); -#if !defined(ARCADIA_BUILD) registerInputFormatAvro(factory); registerOutputFormatAvro(factory); -#endif registerInputFormatArrow(factory); registerOutputFormatArrow(factory); @@ -146,9 +142,7 @@ void registerFormats() registerInputFormatJSONAsString(factory); registerInputFormatLineAsString(factory); -#if !defined(ARCADIA_BUILD) registerInputFormatCapnProto(factory); -#endif registerNonTrivialPrefixAndSuffixCheckerJSONEachRow(factory); registerNonTrivialPrefixAndSuffixCheckerJSONAsString(factory); diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index c53ffa7d59d..2e601888ecc 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -6,9 +6,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index c88acc98825..6b1234f6228 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_BASE64 # include diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index acb5bc8789a..5d87347d8b5 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -39,9 +39,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # pragma GCC diagnostic push diff --git a/src/Functions/FunctionIfBase.h b/src/Functions/FunctionIfBase.h index 17d02de2e27..d0e69ca1ebc 100644 --- a/src/Functions/FunctionIfBase.h +++ b/src/Functions/FunctionIfBase.h @@ -3,9 +3,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB { diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h index da6c71fdfbb..badbde280f1 100644 --- a/src/Functions/FunctionMathBinaryFloat64.h +++ b/src/Functions/FunctionMathBinaryFloat64.h @@ -9,9 +9,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" namespace DB { diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index 85a024d3762..2d39daac366 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -8,9 +8,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" /** FastOps is a fast vector math library from Mikhail Parakhin (former Yandex CTO), * Enabled by default. diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 4c27f090fb2..aaed980c62a 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -20,9 +20,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" namespace DB { diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 568b0c3cc3e..3207b80af71 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -12,9 +12,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # pragma GCC diagnostic push diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 8feb16c86b6..58a7947a135 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL #include diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index fc08ffe4976..0f75750354a 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -18,9 +18,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/Functions/FunctionsHashing.cpp b/src/Functions/FunctionsHashing.cpp index ed977472a5b..3f334e9c302 100644 --- a/src/Functions/FunctionsHashing.cpp +++ b/src/Functions/FunctionsHashing.cpp @@ -30,14 +30,12 @@ void registerFunctionsHashing(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); -#if !defined(ARCADIA_BUILD) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); -#endif #if USE_XXHASH factory.registerFunction(); diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 0e9f54229a5..6b39f2ed5cf 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -6,10 +6,8 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -# include "config_core.h" -#endif +#include "config_functions.h" +#include "config_core.h" #include #include @@ -268,7 +266,6 @@ struct SipHash128Impl } }; -#if !defined(ARCADIA_BUILD) /** Why we need MurmurHash2? * MurmurHash2 is an outdated hash function, superseded by MurmurHash3 and subsequently by CityHash, xxHash, HighwayHash. * Usually there is no reason to use MurmurHash. @@ -390,7 +387,6 @@ struct MurmurHash3Impl128 MurmurHash3_x64_128(begin, size, 0, out_char_data); } }; -#endif /// http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452 /// Care should be taken to do all calculation in unsigned integers (to avoid undefined behaviour on overflow) @@ -1369,14 +1365,12 @@ using FunctionFarmFingerprint64 = FunctionAnyHash; using FunctionFarmHash64 = FunctionAnyHash; using FunctionMetroHash64 = FunctionAnyHash; -#if !defined(ARCADIA_BUILD) using FunctionMurmurHash2_32 = FunctionAnyHash; using FunctionMurmurHash2_64 = FunctionAnyHash; using FunctionGccMurmurHash = FunctionAnyHash; using FunctionMurmurHash3_32 = FunctionAnyHash; using FunctionMurmurHash3_64 = FunctionAnyHash; using FunctionMurmurHash3_128 = FunctionStringHashFixedString; -#endif using FunctionJavaHash = FunctionAnyHash; using FunctionJavaHashUTF16LE = FunctionAnyHash; diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index e861e99861b..105643ff82b 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -43,9 +43,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" namespace DB diff --git a/src/Functions/GCDLCMImpl.h b/src/Functions/GCDLCMImpl.h index 555a67fd462..f4988e3a585 100644 --- a/src/Functions/GCDLCMImpl.h +++ b/src/Functions/GCDLCMImpl.h @@ -6,9 +6,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" namespace DB diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 2c5cd59d05a..12f2f50a5f0 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -16,9 +16,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_EMBEDDED_COMPILER # pragma GCC diagnostic push diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index acedbb0df83..dfa3f00d1cf 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -5,9 +5,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 7dc0712023f..63a14484fe2 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -6,10 +6,8 @@ #include #include "Regexps.h" -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -# include -#endif +#include "config_functions.h" +#include #if USE_RE2_ST # include diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h index 6c652d32786..c2e64671d1f 100644 --- a/src/Functions/MultiMatchAllIndicesImpl.h +++ b/src/Functions/MultiMatchAllIndicesImpl.h @@ -5,10 +5,8 @@ #include #include "Regexps.h" -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -# include -#endif +#include "config_functions.h" +#include #if USE_HYPERSCAN # include diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h index 44a65024864..747c0e5e62c 100644 --- a/src/Functions/MultiMatchAnyImpl.h +++ b/src/Functions/MultiMatchAnyImpl.h @@ -5,10 +5,8 @@ #include #include "Regexps.h" -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -# include -#endif +#include "config_functions.h" +#include #if USE_HYPERSCAN # include diff --git a/src/Functions/RapidJSONParser.h b/src/Functions/RapidJSONParser.h index f864c18ed2b..0e791fe744f 100644 --- a/src/Functions/RapidJSONParser.h +++ b/src/Functions/RapidJSONParser.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_RAPIDJSON # include diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index c18b00088e6..b3477f6f32c 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -14,9 +14,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_HYPERSCAN # include diff --git a/src/Functions/ReplaceRegexpImpl.h b/src/Functions/ReplaceRegexpImpl.h index 46ebab7f5fe..4320f47c424 100644 --- a/src/Functions/ReplaceRegexpImpl.h +++ b/src/Functions/ReplaceRegexpImpl.h @@ -5,10 +5,8 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -# include -#endif +#include "config_functions.h" +#include #if USE_RE2_ST # include diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index 1be9ab32ab2..be85d74619b 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_SIMDJSON # include diff --git a/src/Functions/aes_decrypt_mysql.cpp b/src/Functions/aes_decrypt_mysql.cpp index d5d7a6660e4..764fcf06c1a 100644 --- a/src/Functions/aes_decrypt_mysql.cpp +++ b/src/Functions/aes_decrypt_mysql.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp index bdf7d8d2cce..1d84824d9d6 100644 --- a/src/Functions/aes_encrypt_mysql.cpp +++ b/src/Functions/aes_encrypt_mysql.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL diff --git a/src/Functions/base64Encode.cpp b/src/Functions/base64Encode.cpp index 0c038eb5fca..1f181a31144 100644 --- a/src/Functions/base64Encode.cpp +++ b/src/Functions/base64Encode.cpp @@ -1,9 +1,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_BASE64 # include diff --git a/src/Functions/convertCharset.cpp b/src/Functions/convertCharset.cpp index 184cf5d1b0e..ccfb28ad7ef 100644 --- a/src/Functions/convertCharset.cpp +++ b/src/Functions/convertCharset.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_ICU # include diff --git a/src/Functions/decrypt.cpp b/src/Functions/decrypt.cpp index a9090a31f0e..1cbda0dba99 100644 --- a/src/Functions/decrypt.cpp +++ b/src/Functions/decrypt.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL diff --git a/src/Functions/divide/divide.cpp b/src/Functions/divide/divide.cpp index 5ab11df2a65..cf2cd354a7d 100644 --- a/src/Functions/divide/divide.cpp +++ b/src/Functions/divide/divide.cpp @@ -1,7 +1,7 @@ #include "divide.h" #include -#if defined(__x86_64__) && !defined(ARCADIA_BUILD) +#if defined(__x86_64__) namespace SSE2 { template @@ -25,7 +25,7 @@ namespace Generic template void divideImpl(const A * __restrict a_pos, B b, ResultType * __restrict c_pos, size_t size) { -#if defined(__x86_64__) && !defined(ARCADIA_BUILD) +#if defined(__x86_64__) if (DB::Cpu::CpuFlagsCache::have_AVX2) AVX2::divideImpl(a_pos, b, c_pos, size); else if (DB::Cpu::CpuFlagsCache::have_SSE2) diff --git a/src/Functions/divide/divideImpl.cpp b/src/Functions/divide/divideImpl.cpp index f4c1a97d3ad..6c151dfefb5 100644 --- a/src/Functions/divide/divideImpl.cpp +++ b/src/Functions/divide/divideImpl.cpp @@ -1,14 +1,6 @@ /// This translation unit should be compiled multiple times /// with different values of NAMESPACE and machine flags (sse2, avx2). -#if !defined(NAMESPACE) - #if defined(ARCADIA_BUILD) - #define NAMESPACE Generic - #else - #error "NAMESPACE macro must be defined" - #endif -#endif - #if defined(__AVX2__) #define REG_SIZE 32 #define LIBDIVIDE_AVX2 diff --git a/src/Functions/encrypt.cpp b/src/Functions/encrypt.cpp index ba8ffa1bc59..807443a2fb8 100644 --- a/src/Functions/encrypt.cpp +++ b/src/Functions/encrypt.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 6dab079b484..e7da254117b 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/geoToS2.cpp b/src/Functions/geoToS2.cpp index 15e395b68c4..644e4661412 100644 --- a/src/Functions/geoToS2.cpp +++ b/src/Functions/geoToS2.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index 6c29982bab1..6f018e6aa3f 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index abdc6827122..a50da41620c 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index a4ef21dae79..de4e6ade291 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3GetFaces.cpp b/src/Functions/h3GetFaces.cpp index 4052326495d..5d82c16296c 100644 --- a/src/Functions/h3GetFaces.cpp +++ b/src/Functions/h3GetFaces.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index a212eee169e..5715ceae62d 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index c85670adf35..6bd81b84224 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index b31fb194bc9..e84bc431054 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3IsPentagon.cpp b/src/Functions/h3IsPentagon.cpp index da1dee66b53..039fea39f2a 100644 --- a/src/Functions/h3IsPentagon.cpp +++ b/src/Functions/h3IsPentagon.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index 330f147bd2c..f2f7ae445f2 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index ff0dfabd777..8f21895db01 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index 381dce373bc..30a46bd711a 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3ToGeoBoundary.cpp b/src/Functions/h3ToGeoBoundary.cpp index c55a24acff6..e1e34d1eef3 100644 --- a/src/Functions/h3ToGeoBoundary.cpp +++ b/src/Functions/h3ToGeoBoundary.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 230f8b4796c..d896253822d 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index f00221e480a..25d82d2d40c 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index 80227b5e4e0..857282ec0aa 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index b50add3e9d4..d92ac2aadbb 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/lemmatize.cpp b/src/Functions/lemmatize.cpp index 6d16b032bb5..78f9ee75ed8 100644 --- a/src/Functions/lemmatize.cpp +++ b/src/Functions/lemmatize.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp index 2b3a869e452..2fe6a1159af 100644 --- a/src/Functions/normalizeString.cpp +++ b/src/Functions/normalizeString.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_ICU #include diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 94dfcdb4fda..a76b56fe610 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include "config_core.h" #include @@ -73,9 +71,7 @@ void registerFunctions() registerFunctionsArray(factory); registerFunctionsTuple(factory); registerFunctionsMap(factory); -#if !defined(ARCADIA_BUILD) registerFunctionsBitmap(factory); -#endif registerFunctionsBinaryRepr(factory); registerFunctionsCoding(factory); registerFunctionsCodingUUID(factory); diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index fd55c9cc20a..b24dc4cb9c0 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" namespace DB { diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index dc062ab148a..ffe195973d3 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB { diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp index f2439a3373b..79002f0a97d 100644 --- a/src/Functions/registerFunctionsString.cpp +++ b/src/Functions/registerFunctionsString.cpp @@ -1,7 +1,5 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -# include "config_core.h" -#endif +#include "config_functions.h" +#include "config_core.h" namespace DB { diff --git a/src/Functions/s2CapContains.cpp b/src/Functions/s2CapContains.cpp index e3df7f2350e..c3ebbf0d251 100644 --- a/src/Functions/s2CapContains.cpp +++ b/src/Functions/s2CapContains.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2CapUnion.cpp b/src/Functions/s2CapUnion.cpp index 98080729a50..2328db4cb52 100644 --- a/src/Functions/s2CapUnion.cpp +++ b/src/Functions/s2CapUnion.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2CellsIntersect.cpp b/src/Functions/s2CellsIntersect.cpp index 2ceb13c043b..d7801afe0d0 100644 --- a/src/Functions/s2CellsIntersect.cpp +++ b/src/Functions/s2CellsIntersect.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2GetNeighbors.cpp b/src/Functions/s2GetNeighbors.cpp index f4df8c5a53f..99c1395f3cd 100644 --- a/src/Functions/s2GetNeighbors.cpp +++ b/src/Functions/s2GetNeighbors.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2RectAdd.cpp b/src/Functions/s2RectAdd.cpp index 20ca7c3c940..9a6fcd25e5a 100644 --- a/src/Functions/s2RectAdd.cpp +++ b/src/Functions/s2RectAdd.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2RectContains.cpp b/src/Functions/s2RectContains.cpp index b5cf1ffc505..11db27e68ca 100644 --- a/src/Functions/s2RectContains.cpp +++ b/src/Functions/s2RectContains.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2RectIntersection.cpp b/src/Functions/s2RectIntersection.cpp index b2de333752a..a8a4536c9e7 100644 --- a/src/Functions/s2RectIntersection.cpp +++ b/src/Functions/s2RectIntersection.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2RectUnion.cpp b/src/Functions/s2RectUnion.cpp index d22e250f708..f187c068345 100644 --- a/src/Functions/s2RectUnion.cpp +++ b/src/Functions/s2RectUnion.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2ToGeo.cpp b/src/Functions/s2ToGeo.cpp index fe7c9c76d72..032fdbfe323 100644 --- a/src/Functions/s2ToGeo.cpp +++ b/src/Functions/s2ToGeo.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_S2_GEOMETRY diff --git a/src/Functions/s2_fwd.h b/src/Functions/s2_fwd.h index e3f7026e48c..6e0b58ae118 100644 --- a/src/Functions/s2_fwd.h +++ b/src/Functions/s2_fwd.h @@ -4,12 +4,12 @@ #pragma clang diagnostic ignored "-Wambiguous-reversed-operator" #endif -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include +#include +#include +#include +#include #ifdef __clang__ #pragma clang diagnostic pop diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index 900a6dbf2a9..87d5f955e88 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -11,9 +11,7 @@ # include #endif -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp index 950b1760c23..3da4a7fdd07 100644 --- a/src/Functions/stem.cpp +++ b/src/Functions/stem.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP @@ -11,7 +9,7 @@ #include #include -#include // Y_IGNORE +#include namespace DB diff --git a/src/Functions/stringToH3.cpp b/src/Functions/stringToH3.cpp index 1ecdb456632..1cafd1661f1 100644 --- a/src/Functions/stringToH3.cpp +++ b/src/Functions/stringToH3.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_functions.h" -#endif +#include "config_functions.h" #if USE_H3 diff --git a/src/Functions/synonyms.cpp b/src/Functions/synonyms.cpp index 5cb9e064c40..6d879f9d42c 100644 --- a/src/Functions/synonyms.cpp +++ b/src/Functions/synonyms.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP diff --git a/src/IO/BrotliReadBuffer.cpp b/src/IO/BrotliReadBuffer.cpp index 41991ad0516..b66bbf45054 100644 --- a/src/IO/BrotliReadBuffer.cpp +++ b/src/IO/BrotliReadBuffer.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_BROTLI # include diff --git a/src/IO/BrotliWriteBuffer.cpp b/src/IO/BrotliWriteBuffer.cpp index 3d0afa86360..6d815c3abab 100644 --- a/src/IO/BrotliWriteBuffer.cpp +++ b/src/IO/BrotliWriteBuffer.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_BROTLI # include diff --git a/src/IO/Bzip2ReadBuffer.cpp b/src/IO/Bzip2ReadBuffer.cpp index 098e829188f..df9a8d5b369 100644 --- a/src/IO/Bzip2ReadBuffer.cpp +++ b/src/IO/Bzip2ReadBuffer.cpp @@ -1,10 +1,8 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_BZIP2 # include -# include // Y_IGNORE +# include namespace DB { diff --git a/src/IO/Bzip2WriteBuffer.cpp b/src/IO/Bzip2WriteBuffer.cpp index 6e63779d3cf..033a86b0158 100644 --- a/src/IO/Bzip2WriteBuffer.cpp +++ b/src/IO/Bzip2WriteBuffer.cpp @@ -1,10 +1,8 @@ -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_BZIP2 # include -# include // Y_IGNORE +# include #include diff --git a/src/IO/CompressionMethod.cpp b/src/IO/CompressionMethod.cpp index ef4beef6b62..c61d091cd09 100644 --- a/src/IO/CompressionMethod.cpp +++ b/src/IO/CompressionMethod.cpp @@ -15,9 +15,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include diff --git a/src/IO/FileEncryptionCommon.h b/src/IO/FileEncryptionCommon.h index 91fc80c0b22..28d924e6d81 100644 --- a/src/IO/FileEncryptionCommon.h +++ b/src/IO/FileEncryptionCommon.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_SSL #include diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index 4fcd3576d7b..9f53572fcde 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -9,9 +9,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include @@ -120,7 +118,7 @@ namespace session->setProxyHost(proxy_host); session->setProxyPort(proxy_port); -#if !defined(ARCADIA_BUILD) && defined(POCO_CLICKHOUSE_PATCH) +#if defined(POCO_CLICKHOUSE_PATCH) session->setProxyProtocol(proxy_scheme); /// Turn on tunnel mode if proxy scheme is HTTP while endpoint scheme is HTTPS. diff --git a/src/IO/LZMADeflatingWriteBuffer.cpp b/src/IO/LZMADeflatingWriteBuffer.cpp index 29cde872241..69d4065f0e5 100644 --- a/src/IO/LZMADeflatingWriteBuffer.cpp +++ b/src/IO/LZMADeflatingWriteBuffer.cpp @@ -1,8 +1,6 @@ #include #include -#if !defined(ARCADIA_BUILD) - namespace DB { namespace ErrorCodes @@ -146,4 +144,3 @@ void LZMADeflatingWriteBuffer::finishImpl() } } -#endif diff --git a/src/IO/LZMADeflatingWriteBuffer.h b/src/IO/LZMADeflatingWriteBuffer.h index 98eb1732e76..388bb622df9 100644 --- a/src/IO/LZMADeflatingWriteBuffer.h +++ b/src/IO/LZMADeflatingWriteBuffer.h @@ -3,16 +3,12 @@ #include #include -#if !defined(ARCADIA_BUILD) - #include // Y_IGNORE -#endif +#include namespace DB { -#if !defined(ARCADIA_BUILD) - /// Performs compression using lzma library and writes compressed data to out_ WriteBuffer. class LZMADeflatingWriteBuffer : public BufferWithOwnMemory { @@ -39,26 +35,4 @@ private: bool finished = false; }; -#else - -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} - -class LZMADeflatingWriteBuffer : public BufferWithOwnMemory -{ -public: - LZMADeflatingWriteBuffer( - std::unique_ptr out_ [[maybe_unused]], - int compression_level [[maybe_unused]], - size_t buf_size [[maybe_unused]] = DBMS_DEFAULT_BUFFER_SIZE, - char * existing_memory [[maybe_unused]] = nullptr, - size_t alignment [[maybe_unused]] = 0) - { - throw Exception("LZMADeflatingWriteBuffer is not implemented for arcadia build", ErrorCodes::NOT_IMPLEMENTED); - } -}; - -#endif } diff --git a/src/IO/LZMAInflatingReadBuffer.cpp b/src/IO/LZMAInflatingReadBuffer.cpp index 6a0a7e5ee31..f2df6bdca6a 100644 --- a/src/IO/LZMAInflatingReadBuffer.cpp +++ b/src/IO/LZMAInflatingReadBuffer.cpp @@ -1,6 +1,5 @@ #include -#if !defined(ARCADIA_BUILD) namespace DB { namespace ErrorCodes @@ -88,4 +87,3 @@ bool LZMAInflatingReadBuffer::nextImpl() return true; } } -#endif diff --git a/src/IO/LZMAInflatingReadBuffer.h b/src/IO/LZMAInflatingReadBuffer.h index 46bd5d83183..18922f64516 100644 --- a/src/IO/LZMAInflatingReadBuffer.h +++ b/src/IO/LZMAInflatingReadBuffer.h @@ -3,14 +3,11 @@ #include #include -#if !defined(ARCADIA_BUILD) - #include // Y_IGNORE -#endif +#include namespace DB { -#if !defined(ARCADIA_BUILD) class LZMAInflatingReadBuffer : public BufferWithOwnMemory { public: @@ -31,25 +28,4 @@ private: bool eof; }; -#else - -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} - -class LZMAInflatingReadBuffer : public BufferWithOwnMemory -{ -public: - LZMAInflatingReadBuffer( - std::unique_ptr in_ [[maybe_unused]], - size_t buf_size [[maybe_unused]] = DBMS_DEFAULT_BUFFER_SIZE, - char * existing_memory [[maybe_unused]] = nullptr, - size_t alignment [[maybe_unused]] = 0) - { - throw Exception("LZMADeflatingWriteBuffer is not implemented for arcadia build", ErrorCodes::NOT_IMPLEMENTED); - } -}; - -#endif } diff --git a/src/IO/ReadBufferFromEncryptedFile.h b/src/IO/ReadBufferFromEncryptedFile.h index 9409c4577d2..66768beb28e 100644 --- a/src/IO/ReadBufferFromEncryptedFile.h +++ b/src/IO/ReadBufferFromEncryptedFile.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_SSL #include diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index da8d25c1f6f..e557e3bc324 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index a42404cc631..55fcdca89b3 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -18,9 +18,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 12f5af60ed4..2647e254626 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 @@ -10,10 +8,10 @@ #include #include #include -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include +#include +#include namespace Aws::Http::Standard { diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 12dde929d2c..72774499445 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -1,14 +1,12 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 #include -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include #include #include diff --git a/src/IO/UseSSL.cpp b/src/IO/UseSSL.cpp index 119a3471c07..9ddeb9ccdb5 100644 --- a/src/IO/UseSSL.cpp +++ b/src/IO/UseSSL.cpp @@ -1,8 +1,6 @@ #include "UseSSL.h" -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/IO/WriteBufferFromEncryptedFile.h b/src/IO/WriteBufferFromEncryptedFile.h index 8ae72b405d6..9707343ac4d 100644 --- a/src/IO/WriteBufferFromEncryptedFile.h +++ b/src/IO/WriteBufferFromEncryptedFile.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_SSL #include diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index a8966f7afe1..946ecb45948 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 @@ -14,7 +12,7 @@ # include # include -# include // Y_IGNORE +# include namespace Aws::S3 { diff --git a/src/IO/tests/gtest_file_encryption.cpp b/src/IO/tests/gtest_file_encryption.cpp index 187073c7262..3a114f94ee0 100644 --- a/src/IO/tests/gtest_file_encryption.cpp +++ b/src/IO/tests/gtest_file_encryption.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_SSL #include diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 8111bcb123c..c9aa7144df9 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -5,9 +5,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" namespace DB { diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index d733da4249e..66b105970af 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -19,9 +19,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_JEMALLOC # include diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 93e77b6bde8..7a5c2d638d7 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -41,9 +41,6 @@ using AsynchronousMetricValues = std::unordered_map> servers_to_start_before_tables{nullptr}; diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 40e2ea1bc9b..79956aaebed 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -7,9 +7,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 41bb1f656a7..d42c34f07e2 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -55,11 +55,6 @@ public: String current_query_id; Poco::Net::SocketAddress current_address; -#if defined(ARCADIA_BUILD) - /// This field is only used in foreign "Arcadia" build. - String current_password; -#endif - /// When query_kind == INITIAL_QUERY, these values are equal to current. String initial_user; String initial_query_id; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 69f9518a912..5e68f45da61 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2448,12 +2448,10 @@ void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration } } -#if !defined(ARCADIA_BUILD) if (shared->storage_s3_settings) { shared->storage_s3_settings->loadFromConfig("s3", config); } -#endif } @@ -2490,7 +2488,6 @@ const MergeTreeSettings & Context::getReplicatedMergeTreeSettings() const const StorageS3Settings & Context::getStorageS3Settings() const { -#if !defined(ARCADIA_BUILD) auto lock = getLock(); if (!shared->storage_s3_settings) @@ -2500,9 +2497,6 @@ const StorageS3Settings & Context::getStorageS3Settings() const } return *shared->storage_s3_settings; -#else - throw Exception("S3 is unavailable in Arcadia", ErrorCodes::NOT_IMPLEMENTED); -#endif } void Context::checkCanBeDropped(const String & database, const String & table, const size_t & size, const size_t & max_size_to_drop) const diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6ea810be3c8..cc6df875f7d 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -16,9 +16,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index 6bc23d6cf62..1abaec4a17f 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -8,9 +8,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 08cc260b536..205d7052d0c 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -19,9 +19,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL # include diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index c420885eaa2..c942f33b6df 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -7,9 +7,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" namespace DB diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 571b4ba5e13..90292d17fae 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index c09ab8b78e5..e682a98114d 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -6,9 +6,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL # include diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 5394c1ecaf0..4fbad7e5471 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -12,9 +12,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL # include diff --git a/src/Interpreters/InterpreterExternalDDLQuery.cpp b/src/Interpreters/InterpreterExternalDDLQuery.cpp index e91b95a988d..ac3f18f1741 100644 --- a/src/Interpreters/InterpreterExternalDDLQuery.cpp +++ b/src/Interpreters/InterpreterExternalDDLQuery.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 83061de2c10..e34d974fa80 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -44,9 +44,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" namespace DB { diff --git a/src/Interpreters/JIT/CHJIT.h b/src/Interpreters/JIT/CHJIT.h index 0a553be6bb3..58da0fcffcc 100644 --- a/src/Interpreters/JIT/CHJIT.h +++ b/src/Interpreters/JIT/CHJIT.h @@ -1,17 +1,15 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER #include #include -#include // Y_IGNORE -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include +#include namespace DB { diff --git a/src/Interpreters/JIT/CompileDAG.h b/src/Interpreters/JIT/CompileDAG.h index ee9cc5cc83a..84bfc738cfc 100644 --- a/src/Interpreters/JIT/CompileDAG.h +++ b/src/Interpreters/JIT/CompileDAG.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER diff --git a/src/Interpreters/JIT/CompiledExpressionCache.h b/src/Interpreters/JIT/CompiledExpressionCache.h index 7d20627d5d2..936b5798280 100644 --- a/src/Interpreters/JIT/CompiledExpressionCache.h +++ b/src/Interpreters/JIT/CompiledExpressionCache.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER # include diff --git a/src/Interpreters/JIT/compileFunction.h b/src/Interpreters/JIT/compileFunction.h index ae91c2a7650..e35ea49291c 100644 --- a/src/Interpreters/JIT/compileFunction.h +++ b/src/Interpreters/JIT/compileFunction.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER diff --git a/src/Interpreters/Lemmatizers.cpp b/src/Interpreters/Lemmatizers.cpp index 78af43285ef..72f0161b8fd 100644 --- a/src/Interpreters/Lemmatizers.cpp +++ b/src/Interpreters/Lemmatizers.cpp @@ -1,13 +1,10 @@ - -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP #include -#include // Y_IGNORE -#include // Y_IGNORE +#include +#include #include #include diff --git a/src/Interpreters/Lemmatizers.h b/src/Interpreters/Lemmatizers.h index 3712f76387b..936c796bd74 100644 --- a/src/Interpreters/Lemmatizers.h +++ b/src/Interpreters/Lemmatizers.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP diff --git a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp index 08af123448e..6055d53cf11 100644 --- a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp +++ b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 865c9551219..a652ee7e81a 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -324,12 +324,6 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->current_user = credentials_.getUserName(); prepared_client_info->current_address = address; - -#if defined(ARCADIA_BUILD) - /// This is harmful field that is used only in foreign "Arcadia" build. - if (const auto * basic_credentials = dynamic_cast(&credentials_)) - prepared_client_info->current_password = basic_credentials->getPassword(); -#endif } ClientInfo & Session::getClientInfo() @@ -456,9 +450,6 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t { res_client_info.current_user = prepared_client_info->current_user; res_client_info.current_address = prepared_client_info->current_address; -#if defined(ARCADIA_BUILD) - res_client_info.current_password = prepared_client_info->current_password; -#endif } /// Set parameters of initial query. diff --git a/src/Interpreters/SynonymsExtensions.cpp b/src/Interpreters/SynonymsExtensions.cpp index 6147fa14674..ddc90d04e02 100644 --- a/src/Interpreters/SynonymsExtensions.cpp +++ b/src/Interpreters/SynonymsExtensions.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP @@ -11,7 +9,7 @@ #include #include -#include // Y_IGNORE +#include namespace DB { diff --git a/src/Interpreters/SynonymsExtensions.h b/src/Interpreters/SynonymsExtensions.h index d1f3548ce64..c4a70b28c60 100644 --- a/src/Interpreters/SynonymsExtensions.h +++ b/src/Interpreters/SynonymsExtensions.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_NLP diff --git a/src/Interpreters/examples/jit_example.cpp b/src/Interpreters/examples/jit_example.cpp index 3fa2a901eac..d4613b7fad7 100644 --- a/src/Interpreters/examples/jit_example.cpp +++ b/src/Interpreters/examples/jit_example.cpp @@ -1,8 +1,6 @@ #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_EMBEDDED_COMPILER diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 95fb8d38454..f401f708ab1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -393,10 +393,8 @@ static std::tuple executeQueryImpl( client_info.initial_query_start_time_microseconds = time_in_microseconds(current_time); } -#if !defined(ARCADIA_BUILD) assert(internal || CurrentThread::get().getQueryContext()); assert(internal || CurrentThread::get().getQueryContext()->getCurrentQueryId() == CurrentThread::getQueryId()); -#endif const Settings & settings = context->getSettingsRef(); diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 9e0c46af730..419c22dd0d5 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -3,9 +3,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" namespace DB diff --git a/src/Parsers/ParserExternalDDLQuery.cpp b/src/Parsers/ParserExternalDDLQuery.cpp index b6ada39a800..c64c50cd99f 100644 --- a/src/Parsers/ParserExternalDDLQuery.cpp +++ b/src/Parsers/ParserExternalDDLQuery.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h index 705c47c9b17..1136937e65b 100644 --- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_ARROW diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h index 44d46e97d2a..154292d838f 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_ARROW diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.h b/src/Processors/Formats/Impl/ArrowBufferedStreams.h index a49936f326c..a91274e5ad1 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.h +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_ARROW || USE_ORC || USE_PARQUET diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index abce6759250..5c495901517 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_ARROW || USE_ORC || USE_PARQUET diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 8734f9c7279..014715ba9ee 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -468,13 +468,11 @@ namespace DB fillArrowArrayWithDecimalColumnData(column, null_bytemap, array_builder, format_name, start, end); return true; } -#if !defined(ARCADIA_BUILD) if constexpr (std::is_same_v>) { fillArrowArrayWithDecimalColumnData(column, null_bytemap, array_builder, format_name, start, end); return true; } -#endif return false; }; diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h index c0885d3778c..1fb2a8af65e 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h @@ -1,8 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif - +#include "config_formats.h" #if USE_ARROW || USE_PARQUET diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index 1d790b4485b..fa5c2e74584 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -1,10 +1,7 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -# include "config_core.h" -#endif - +#include "config_formats.h" +#include "config_core.h" #if USE_MSGPACK diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h index 9c66bb9d207..2766eb144e4 100644 --- a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h @@ -1,9 +1,7 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -# include "config_core.h" -#endif +#include "config_formats.h" +#include "config_core.h" #if USE_MSGPACK diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h index 60ce963a6a1..254d0554cb2 100644 --- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_ORC #include diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index 557bf6cc07a..c131d724450 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_formats.h" -#endif #if USE_ORC #include diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index a38db0b3a02..bae380f2c80 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PARQUET #include diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index ba82000e463..80d11b540b1 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PARQUET # include diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h index b2eabd4f37c..553a2dfd7f0 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PROTOBUF # include diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index 54324490a3b..5323aa56323 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_formats.h" -#endif +#include "config_formats.h" #if USE_PROTOBUF # include diff --git a/src/Processors/Sources/MySQLSource.cpp b/src/Processors/Sources/MySQLSource.cpp index 5bda662466c..8e9cdcfda48 100644 --- a/src/Processors/Sources/MySQLSource.cpp +++ b/src/Processors/Sources/MySQLSource.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_MYSQL #include diff --git a/src/Processors/Sources/SQLiteSource.h b/src/Processors/Sources/SQLiteSource.h index 0f8b42c536b..28106062fb5 100644 --- a/src/Processors/Sources/SQLiteSource.h +++ b/src/Processors/Sources/SQLiteSource.h @@ -1,14 +1,12 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_SQLITE #include #include -#include // Y_IGNORE +#include namespace DB diff --git a/src/Processors/Transforms/PostgreSQLSource.h b/src/Processors/Transforms/PostgreSQLSource.h index 2736afec7a9..c7e55c09c32 100644 --- a/src/Processors/Transforms/PostgreSQLSource.h +++ b/src/Processors/Transforms/PostgreSQLSource.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX #include diff --git a/src/Server/GRPCServer.h b/src/Server/GRPCServer.h index ef86b902b5a..25c3813c11d 100644 --- a/src/Server/GRPCServer.h +++ b/src/Server/GRPCServer.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_GRPC #include diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index cba0b6f0592..8d51b30138c 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -8,9 +8,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index b7e7ac2ac33..b2a00055532 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -35,9 +35,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include #include @@ -495,10 +493,7 @@ void HTTPHandler::processQuery( } // Parse the OpenTelemetry traceparent header. - // Disable in Arcadia -- it interferes with the - // test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test. ClientInfo client_info = session->getClientInfo(); -#if !defined(ARCADIA_BUILD) if (request.has("traceparent")) { std::string opentelemetry_traceparent = request.get("traceparent"); @@ -512,7 +507,6 @@ void HTTPHandler::processQuery( } client_info.client_trace_context.tracestate = request.get("tracestate", ""); } -#endif auto context = session->makeQueryContext(std::move(client_info)); diff --git a/src/Server/KeeperTCPHandler.h b/src/Server/KeeperTCPHandler.h index 274fb21af63..9983749f145 100644 --- a/src/Server/KeeperTCPHandler.h +++ b/src/Server/KeeperTCPHandler.h @@ -1,9 +1,7 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include -# include "config_core.h" -#endif +#include +#include "config_core.h" #if USE_NURAFT diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 8f4f04e56c5..2ba5359ca31 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -21,9 +21,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index 12be8c12921..7ef212bf36e 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -9,9 +9,7 @@ #include #include "IServer.h" -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/Server/MySQLHandlerFactory.h b/src/Server/MySQLHandlerFactory.h index df7bd794b16..106fdfdf341 100644 --- a/src/Server/MySQLHandlerFactory.h +++ b/src/Server/MySQLHandlerFactory.h @@ -5,9 +5,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 472bf421ce7..fee4ace3452 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -10,9 +10,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #if USE_SSL # include diff --git a/src/Server/ProtocolServerAdapter.h b/src/Server/ProtocolServerAdapter.h index 2e3d67d081f..04c46b53356 100644 --- a/src/Server/ProtocolServerAdapter.h +++ b/src/Server/ProtocolServerAdapter.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #include #include diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 64f7cc0ae09..3c36321b936 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -53,9 +53,7 @@ #include "Core/Protocol.h" #include "TCPHandler.h" -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace CurrentMetrics { diff --git a/src/Storages/HDFS/HDFSCommon.h b/src/Storages/HDFS/HDFSCommon.h index 0c007afdcfe..877bf7dd74f 100644 --- a/src/Storages/HDFS/HDFSCommon.h +++ b/src/Storages/HDFS/HDFSCommon.h @@ -1,15 +1,13 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_HDFS #include #include #include -#include // Y_IGNORE +#include #include #include diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.h b/src/Storages/HDFS/ReadBufferFromHDFS.h index 035a55bd0fa..09cf4d290fd 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.h +++ b/src/Storages/HDFS/ReadBufferFromHDFS.h @@ -1,15 +1,13 @@ #pragma once -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #if USE_HDFS #include #include #include #include -#include // Y_IGNORE +#include #include #include #include diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 6ff8bb3f13f..9020a091782 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_HDFS diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.h b/src/Storages/HDFS/WriteBufferFromHDFS.h index 174a308d3ed..e9ee2fc2d51 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.h +++ b/src/Storages/HDFS/WriteBufferFromHDFS.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) - #include -#endif +#include #if USE_HDFS #include diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h b/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h index cc147a01d32..b729515032f 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLSettings.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX #include diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h index 4b1fa1d5b04..cebb3b89f8e 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX #include "PostgreSQLReplicationHandler.h" diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.h b/src/Storages/ReadFinalForExternalReplicaStorage.h index 1be0aa0f4a6..979945a38c7 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.h +++ b/src/Storages/ReadFinalForExternalReplicaStorage.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL || USE_LIBPQXX diff --git a/src/Storages/StorageExternalDistributed.h b/src/Storages/StorageExternalDistributed.h index c29bfcd891f..33a58d324da 100644 --- a/src/Storages/StorageExternalDistributed.h +++ b/src/Storages/StorageExternalDistributed.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #include #include diff --git a/src/Storages/StorageMaterializedMySQL.cpp b/src/Storages/StorageMaterializedMySQL.cpp index 37a9838593b..c214540151a 100644 --- a/src/Storages/StorageMaterializedMySQL.cpp +++ b/src/Storages/StorageMaterializedMySQL.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL diff --git a/src/Storages/StorageMaterializedMySQL.h b/src/Storages/StorageMaterializedMySQL.h index 95ef4ad97fa..8ba98c3000f 100644 --- a/src/Storages/StorageMaterializedMySQL.h +++ b/src/Storages/StorageMaterializedMySQL.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_MYSQL diff --git a/src/Storages/StorageMongoDBSocketFactory.cpp b/src/Storages/StorageMongoDBSocketFactory.cpp index 9a2b120e9ed..f21e8746eb9 100644 --- a/src/Storages/StorageMongoDBSocketFactory.cpp +++ b/src/Storages/StorageMongoDBSocketFactory.cpp @@ -2,9 +2,7 @@ #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include #include #include diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index 134c9939e3e..cc3673e50ca 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_MYSQL diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index 10a60bf9b21..7d8752c91b9 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX #include diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index 9d05235552c..f49fd35044d 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -1,8 +1,6 @@ #include "Storages/StorageS3Cluster.h" -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index b9ce0a15dfa..d1e02c5a730 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/Storages/StorageSQLite.h b/src/Storages/StorageSQLite.h index 47ed40bef75..dcc17322a1a 100644 --- a/src/Storages/StorageSQLite.h +++ b/src/Storages/StorageSQLite.h @@ -1,14 +1,12 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_SQLITE #include #include -#include // Y_IGNORE +#include namespace Poco { diff --git a/src/Storages/System/IStorageSystemOneBlock.h b/src/Storages/System/IStorageSystemOneBlock.h index 6897b4a3a79..9b51bd73903 100644 --- a/src/Storages/System/IStorageSystemOneBlock.h +++ b/src/Storages/System/IStorageSystemOneBlock.h @@ -31,12 +31,7 @@ class IStorageSystemOneBlock : public IStorage protected: virtual void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const = 0; - public: -#if defined(ARCADIA_BUILD) - IStorageSystemOneBlock(const String & name_) : IStorageSystemOneBlock(StorageID{"system", name_}) {} -#endif - IStorageSystemOneBlock(const StorageID & table_id_) : IStorage(table_id_) { StorageInMemoryMetadata metadata_; diff --git a/src/Storages/System/StorageSystemAsynchronousMetrics.h b/src/Storages/System/StorageSystemAsynchronousMetrics.h index de8b9b93378..2efe008ef3a 100644 --- a/src/Storages/System/StorageSystemAsynchronousMetrics.h +++ b/src/Storages/System/StorageSystemAsynchronousMetrics.h @@ -25,12 +25,6 @@ private: const AsynchronousMetrics & async_metrics; protected: -#if defined(ARCADIA_BUILD) - StorageSystemAsynchronousMetrics(const String & name_, const AsynchronousMetrics & async_metrics_) - : StorageSystemAsynchronousMetrics(StorageID{"system", name_}, async_metrics_) - { - } -#endif StorageSystemAsynchronousMetrics(const StorageID & table_id_, const AsynchronousMetrics & async_metrics_); void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo & query_info) const override; diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp b/src/Storages/System/StorageSystemBuildOptions.cpp index 01a60a0235c..83cd4b036da 100644 --- a/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/src/Storages/System/StorageSystemBuildOptions.cpp @@ -18,15 +18,11 @@ NamesAndTypesList StorageSystemBuildOptions::getNamesAndTypes() void StorageSystemBuildOptions::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const { -#if !defined(ARCADIA_BUILD) for (auto * it = auto_config_build; *it; it += 2) { res_columns[0]->insert(it[0]); res_columns[1]->insert(it[1]); } -#else - UNUSED(res_columns); -#endif } } diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 4950a742dc9..30fe38f4936 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #include #include @@ -21,7 +19,6 @@ #include #include #include - #include #include #include @@ -49,15 +46,11 @@ #include #include #include - -#if !defined(ARCADIA_BUILD) - #include - #include -#endif +#include +#include #include #include #include - #include #include #include @@ -128,10 +121,8 @@ void attachSystemTablesLocal(IDatabase & system_database) attach(system_database, "errors"); attach(system_database, "warnings"); attach(system_database, "data_skipping_indices"); -#if !defined(ARCADIA_BUILD) attach(system_database, "licenses"); attach(system_database, "time_zones"); -#endif #ifdef OS_LINUX attach(system_database, "stack_trace"); #endif diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 9f6c18f53d3..51286af6697 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -1,10 +1,8 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -# include "config_core.h" -#endif +#include +#include "config_core.h" namespace DB { diff --git a/src/TableFunctions/ITableFunctionXDBC.h b/src/TableFunctions/ITableFunctionXDBC.h index 4f6656773b1..ddeaeaa9b10 100644 --- a/src/TableFunctions/ITableFunctionXDBC.h +++ b/src/TableFunctions/ITableFunctionXDBC.h @@ -5,9 +5,7 @@ #include #include -#if !defined(ARCADIA_BUILD) -# include -#endif +#include namespace DB { diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 80f108eb68a..b7c6ee8803e 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_MYSQL #include diff --git a/src/TableFunctions/TableFunctionMySQL.h b/src/TableFunctions/TableFunctionMySQL.h index 78e84dfed29..876dd43f598 100644 --- a/src/TableFunctions/TableFunctionMySQL.h +++ b/src/TableFunctions/TableFunctionMySQL.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include "config_core.h" -#endif +#include "config_core.h" #if USE_MYSQL #include diff --git a/src/TableFunctions/TableFunctionPostgreSQL.h b/src/TableFunctions/TableFunctionPostgreSQL.h index ff363e3a6cf..7b33998a967 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.h +++ b/src/TableFunctions/TableFunctionPostgreSQL.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_LIBPQXX #include diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index 7e28decfdb0..aa3ae20b61d 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -1,6 +1,4 @@ -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/TableFunctions/TableFunctionS3Cluster.h b/src/TableFunctions/TableFunctionS3Cluster.h index 3f2be12497c..cc857725ce6 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.h +++ b/src/TableFunctions/TableFunctionS3Cluster.h @@ -1,8 +1,6 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include -#endif #if USE_AWS_S3 diff --git a/src/TableFunctions/TableFunctionSQLite.h b/src/TableFunctions/TableFunctionSQLite.h index 4649dac016f..e80e04260bc 100644 --- a/src/TableFunctions/TableFunctionSQLite.h +++ b/src/TableFunctions/TableFunctionSQLite.h @@ -1,7 +1,5 @@ #pragma once -#if !defined(ARCADIA_BUILD) #include "config_core.h" -#endif #if USE_SQLITE #include diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 5b01f284c95..8dbb5ebb5fa 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -1,9 +1,7 @@ #pragma once -#if !defined(ARCADIA_BUILD) -# include -# include "config_core.h" -#endif +#include +#include "config_core.h" namespace DB { From acdcb53f1821648c4d417d4c078c65e4237055be Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 28 Oct 2021 09:25:34 +0300 Subject: [PATCH 164/173] Update Settings.h --- src/Core/Settings.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b4c5f8fff59..7aade8e2d0f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -541,6 +541,7 @@ class IColumn; M(Bool, database_replicated_ddl_output, true, "Obsolete setting, does nothing.", 0) \ M(UInt64, replication_alter_columns_timeout, 60, "Obsolete setting, does nothing.", 0) \ M(Bool, optimize_fuse_sum_count_avg, false, "Obsolete, use optimize_syntax_fuse_functions", 0) \ + M(UInt64, odbc_max_field_size, 0, "Obsolete setting, does nothing.", 0) \ /** The section above is for obsolete settings. Do not add anything there. */ From 8956480f08281b017fe679d3c981cf37c5e53452 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 28 Oct 2021 11:23:35 +0300 Subject: [PATCH 165/173] Update adopters.md --- docs/en/introduction/adopters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 6dd61ac6ad4..6dd53fffb86 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -24,7 +24,7 @@ toc_title: Adopters | ArenaData | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) | | Argedor | ClickHouse support | — | — | — | [Official website](https://www.argedor.com/en/clickhouse/) | | Avito | Classifieds | Monitoring | — | — | [Meetup, April 2020](https://www.youtube.com/watch?v=n1tm4j4W8ZQ) | -| Badoo | Dating | Timeseries | — | — | [Slides in Russian, December 2019](https://presentations.clickhouse.com/meetup38/forecast.pdf) | +| Badoo | Dating | Timeseries | — | 1.6 mln events/sec (2018) | [Slides in Russian, December 2019](https://presentations.clickhouse.com/meetup38/forecast.pdf) | | Beeline | Telecom | Data Platform | — | — | [Blog post, July 2021](https://habr.com/en/company/beeline/blog/567508/) | | Benocs | Network Telemetry and Analytics | Main Product | — | — | [Slides in English, October 2017](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup9/lpm.pdf) | | BIGO | Video | Computing Platform | — | — | [Blog Article, August 2020](https://www.programmersought.com/article/44544895251/) | From c9a1cc53d390ea47c1118f58d4b8e270a925e9b1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 28 Oct 2021 11:48:59 +0300 Subject: [PATCH 166/173] Update Context.cpp --- src/Interpreters/Context.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 5e68f45da61..983e2c5afea 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -130,7 +130,6 @@ namespace ErrorCodes extern const int NO_ELEMENTS_IN_CONFIG; extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int LOGICAL_ERROR; - extern const int NOT_IMPLEMENTED; extern const int INVALID_SETTING_VALUE; extern const int UNKNOWN_READ_METHOD; } From f0386403dd45fb88c83bcc6eda026e2855f890f1 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Thu, 28 Oct 2021 11:56:20 +0300 Subject: [PATCH 167/173] new example --- docs/en/operations/system-tables/replicas.md | 38 ++++++++++---------- docs/ru/operations/system-tables/replicas.md | 38 ++++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index c959100b6e4..d00bb9dd79f 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -8,7 +8,7 @@ Example: ``` sql SELECT * FROM system.replicas -WHERE table = 'visits' +WHERE table = 'test_table' FORMAT Vertical ``` @@ -16,9 +16,9 @@ FORMAT Vertical Query id: dc6dcbcb-dc28-4df9-ae27-4354f5b3b13e Row 1: -────── -database: default -table: test +─────── +database: db +table: test_table engine: ReplicatedMergeTree is_leader: 1 can_become_leader: 1 @@ -26,30 +26,30 @@ is_readonly: 0 is_session_expired: 0 future_parts: 0 parts_to_check: 0 -zookeeper_path: /test/t -replica_name: 1 -replica_path: /test/t/replicas/1 +zookeeper_path: /test/test_table +replica_name: r1 +replica_path: /test/test_table/replicas/r1 columns_version: -1 -queue_size: 0 -inserts_in_queue: 0 +queue_size: 27 +inserts_in_queue: 27 merges_in_queue: 0 part_mutations_in_queue: 0 -queue_oldest_time: 1970-01-01 03:00:00 -inserts_oldest_time: 1970-01-01 03:00:00 +queue_oldest_time: 2021-10-12 14:48:48 +inserts_oldest_time: 2021-10-12 14:48:48 merges_oldest_time: 1970-01-01 03:00:00 part_mutations_oldest_time: 1970-01-01 03:00:00 -oldest_part_to_get: +oldest_part_to_get: 1_17_17_0 oldest_part_to_merge_to: oldest_part_to_mutate_to: -log_max_index: 0 -log_pointer: 0 -last_queue_update: 1970-01-01 03:00:00 -absolute_delay: 0 -total_replicas: 1 -active_replicas: 1 +log_max_index: 206 +log_pointer: 207 +last_queue_update: 2021-10-12 14:50:08 +absolute_delay: 99 +total_replicas: 5 +active_replicas: 5 last_queue_update_exception: zookeeper_exception: -replica_is_active: {'1':1} +replica_is_active: {'r1':1,'r2':1} ``` Columns: diff --git a/docs/ru/operations/system-tables/replicas.md b/docs/ru/operations/system-tables/replicas.md index c7bc9c5982e..8f86f94dedd 100644 --- a/docs/ru/operations/system-tables/replicas.md +++ b/docs/ru/operations/system-tables/replicas.md @@ -8,7 +8,7 @@ ``` sql SELECT * FROM system.replicas -WHERE table = 'visits' +WHERE table = 'test_table' FORMAT Vertical ``` @@ -16,9 +16,9 @@ FORMAT Vertical Query id: dc6dcbcb-dc28-4df9-ae27-4354f5b3b13e Row 1: -────── -database: default -table: test +─────── +database: db +table: test_table engine: ReplicatedMergeTree is_leader: 1 can_become_leader: 1 @@ -26,30 +26,30 @@ is_readonly: 0 is_session_expired: 0 future_parts: 0 parts_to_check: 0 -zookeeper_path: /test/t -replica_name: 1 -replica_path: /test/t/replicas/1 +zookeeper_path: /test/test_table +replica_name: r1 +replica_path: /test/test_table/replicas/r1 columns_version: -1 -queue_size: 0 -inserts_in_queue: 0 +queue_size: 27 +inserts_in_queue: 27 merges_in_queue: 0 part_mutations_in_queue: 0 -queue_oldest_time: 1970-01-01 03:00:00 -inserts_oldest_time: 1970-01-01 03:00:00 +queue_oldest_time: 2021-10-12 14:48:48 +inserts_oldest_time: 2021-10-12 14:48:48 merges_oldest_time: 1970-01-01 03:00:00 part_mutations_oldest_time: 1970-01-01 03:00:00 -oldest_part_to_get: +oldest_part_to_get: 1_17_17_0 oldest_part_to_merge_to: oldest_part_to_mutate_to: -log_max_index: 0 -log_pointer: 0 -last_queue_update: 1970-01-01 03:00:00 -absolute_delay: 0 -total_replicas: 1 -active_replicas: 1 +log_max_index: 206 +log_pointer: 207 +last_queue_update: 2021-10-12 14:50:08 +absolute_delay: 99 +total_replicas: 5 +active_replicas: 5 last_queue_update_exception: zookeeper_exception: -replica_is_active: {'1':1} +replica_is_active: {'r1':1,'r2':1} ``` Столбцы: From e59c51d76b58507d6232011330b55871256ba4da Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Thu, 28 Oct 2021 08:01:01 -0600 Subject: [PATCH 168/173] Add funding press release (#30806) * Add blog post for funding announcement * Update homepage hero for funding press release * Add full list of investors to company page * Add press release url * Update website/templates/company/investors.html Co-authored-by: Ivan Blinkov * Update website/templates/company/investors.html Co-authored-by: Ivan Blinkov * Update website/blog/en/2021/clickhouse-raises-250m-series-b.md Co-authored-by: Ivan Blinkov * Update website/templates/index/hero.html Co-authored-by: Ivan Blinkov * Update website/templates/index/hero.html Co-authored-by: Ivan Blinkov Co-authored-by: Ivan Blinkov --- .../2021/clickhouse-raises-250m-series-b.md | 15 + website/css/main.css | 2 +- website/images/logos/logo-almaz-capital.svg | 1 + .../images/logos/logo-altimeter-capital.png | Bin 0 -> 3600 bytes website/images/logos/logo-coatue.png | Bin 0 -> 6128 bytes website/images/logos/logo-firstmark.svg | 579 ++++++++++++++++++ .../images/logos/logo-leadedge-capital.png | Bin 0 -> 3029 bytes website/images/logos/logo-lightspeed.png | Bin 0 -> 3751 bytes website/images/logos/logo-redpoint.png | Bin 0 -> 6109 bytes website/images/logos/logo-yandex.png | Bin 4152 -> 8381 bytes website/images/photos/caryn-marooney.jpg | Bin 0 -> 27066 bytes website/images/photos/kevin-wang.jpg | Bin 0 -> 22029 bytes website/src/scss/components/_logo.scss | 8 + website/templates/company/investors.html | 41 +- website/templates/index/hero.html | 15 +- 15 files changed, 649 insertions(+), 12 deletions(-) create mode 100644 website/blog/en/2021/clickhouse-raises-250m-series-b.md create mode 100644 website/images/logos/logo-almaz-capital.svg create mode 100644 website/images/logos/logo-altimeter-capital.png create mode 100644 website/images/logos/logo-coatue.png create mode 100644 website/images/logos/logo-firstmark.svg create mode 100644 website/images/logos/logo-leadedge-capital.png create mode 100644 website/images/logos/logo-lightspeed.png create mode 100644 website/images/logos/logo-redpoint.png create mode 100644 website/images/photos/caryn-marooney.jpg create mode 100644 website/images/photos/kevin-wang.jpg create mode 100644 website/src/scss/components/_logo.scss diff --git a/website/blog/en/2021/clickhouse-raises-250m-series-b.md b/website/blog/en/2021/clickhouse-raises-250m-series-b.md new file mode 100644 index 00000000000..5933782c924 --- /dev/null +++ b/website/blog/en/2021/clickhouse-raises-250m-series-b.md @@ -0,0 +1,15 @@ +--- +title: 'ClickHouse raises a $250M Series B at a $2B valuation...and we are hiring' +image: 'https://blog-images.clickhouse.com/en/2021/clickhouse-raises-250m-series-b/featured.jpg' +date: '2021-10-28' +author: '[Dorota Szeremeta](https://www.linkedin.com/in/dorota-szeremeta-a849b7/)' +tags: ['company', 'community'] +--- + +Today, we are excited to announce that ClickHouse has raised $250 million in Series B funding at a $2B valuation. The round was led by Coatue and Altimeter, with participation from existing investors Index, Benchmark, Yandex, Almaz, and Lead Edge alongside new investors Lightspeed, Redpoint, and FirstMark. We are thrilled to partner with such an accomplished group. Not only is it a testament to the disruptive potential of our technology and our momentum to date, but more importantly, it’s an aggressive investment in our future. This funding will enable ClickHouse to grow our team, scale the product, and continue building a world-class, commercial-grade cloud solution that’s secure, compliant, and convenient for any customer to use. (You can read more about the funding round in our press release here). + +For the past five years, ClickHouse thrived as a popular open-source product— with a dedicated user community. ClickHouse thrived for a simple reason—we deliver high query processing speed and data storage efficiency that is unmatched. ClickHouse is capable of processing 100+ PBs of data with more than 100 billion records inserted every day. It is 100-1000X faster than traditional approaches and this speed and efficiency has attracted customers that are household names. The commercial potential for the product was clear - and in September 2021, we decided to combine resources and officially incorporate as a company. We took the team of core ClickHouse developers led by ClickHouse creator [Alexey Milovidov](https://www.linkedin.com/in/alexey-milovidov-7b658135/) and brought in a world-class business team led by [Aaron Katz](https://www.linkedin.com/in/aaron-katz-5762094/) and a cloud engineering team led by [Yury Izrailevsky](https://www.linkedin.com/in/yuryizrailevsky/). + +Fueled by this large investment, we are hiring. We plan to double our team this year, and double it again next year. We are calling on all visionaries, builders, and contributors to join us as we build ClickHouse into a transformative, paradigm-shifting company. Our mission is to become the first choice of analytical database management systems. We are relentlessly focused on building an enterprise-grade, highly secure, and fault tolerant, multi-tenant service in the cloud that is accessible to organizations across sizes and sectors. + +If this sounds like something you want to be a part of, now’s the time to join us. To learn more about the positions we are hiring for, and what we are looking for in new hires, visit [www.clickhouse.com/careers](www.clickhouse.com/careers) diff --git a/website/css/main.css b/website/css/main.css index 1d0b6f5c116..70a73dda521 100644 --- a/website/css/main.css +++ b/website/css/main.css @@ -1 +1 @@ -@media screen and (max-width:978.98px){.btn{padding:8px 16px}}@media screen and (max-width:978.98px){.btn-lg{padding:12px 24px}}.btn-primary,.btn-primary:active,.btn-primary:hover{color:#212529}.btn-outline-primary{background:#fffaf0;border-color:#fc0;color:#212529}.btn-outline-primary:active,.btn-outline-primary:hover{background:#fc0;border-color:#fc0;color:#212529}.btn-secondary{border-color:#212529;color:#fff}.btn-outline-secondary,.btn-secondary:active,.btn-secondary:hover{background:#fff;border-color:#212529;color:#212529}.btn-outline-secondary:active,.btn-outline-secondary:hover{background:#212529;border-color:#212529;color:#fff}.btn-tertiary{border-color:#257af4;color:#fff}.btn-tertiary:active,.btn-tertiary:hover{background:#257af4;border-color:#257af4;color:#fff}.btn-outline-tertiary{background:#e3f1fe;color:#257af4}.btn-outline-tertiary:active,.btn-outline-tertiary:hover{background:#257af4;color:#fff}.btns{align-items:center;display:flex;justify-content:center}.btns .btn+.btn{margin-left:24px}.btns .btn-lg+.btn-lg{margin-left:40px}.card{box-shadow:0 8px 20px rgba(108,117,125,.2);overflow:hidden;transition:box-shadow .2s,transform .2s;width:100%}.card,.card-body{position:relative}.card-body{z-index:10}.card.is-large .card-body{padding:40px}.card.bg-primary-light{border-color:#fc0}.card.has-dark-border{border-color:#6c757d}.card.has-pattern:after,.card.has-pattern:before{background-repeat:no-repeat;background-size:auto 100%;bottom:0;content:"";display:block;position:absolute;top:0;width:72px}.card.has-pattern:before{background-image:url(../images/backgrounds/bg-card-pattern-blue-1.png);background-position:0 0;left:0}.card.has-pattern:after{background-image:url(../images/backgrounds/bg-card-pattern-blue-2.png);background-position:100% 0;right:0}.card.has-hover:active,.card.has-hover:hover,a.card:active,a.card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2);transform:translateY(-8px)}.card.has-highlight:after,.card.has-hover:after,a.card:after{content:"";display:block;height:8px;margin-top:auto;transition:background .2s;width:100%}.card.has-highlight:after,.card.has-hover:active:after,.card.has-hover:hover:after,a.card:active:after,a.card:hover:after{background:#e3f1fe}.case-study-cards{-moz-column-gap:40px;column-gap:40px;display:grid;grid-template-columns:1fr;row-gap:40px;padding-bottom:40px;position:relative}.case-study-cards>div{align-items:stretch;display:flex}.case-study-cards:before{background:#d6dbdf;bottom:0;content:"";display:block;left:20px;position:absolute;top:40px;width:100vw}@media screen and (min-width:980px){.case-study-cards{grid-template-columns:repeat(2,minmax(0,1fr));row-gap:80px;padding-bottom:120px}.case-study-cards:before{left:-40px;top:120px}}.case-study-card{align-items:stretch;flex-direction:row;flex-shrink:0;left:0;transition:box-shadow .2s,left .4s,width .4s,z-index 0s;transition-delay:0s,.6s,.6s,0s;width:100%;z-index:2}@media screen and (max-width:979.98px){.case-study-card .row{min-height:0!important}}@media screen and (min-width:980px){.case-study-card:active,.case-study-card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2)}.case-study-card:not(.is-open){cursor:pointer}.case-study-card.is-open{transform:none!important;transition-delay:0s,0s,0s,0s;width:calc(200% + 40px);z-index:10}.case-study-card.is-closing{z-index:10}.case-study-card.open-left.is-open{left:calc(-100% - 40px)}.case-study-card:before{background:no-repeat url(../images/backgrounds/bg-card-pattern-red.png);background-position:100%;background-size:contain;content:"";display:block;height:calc(100% - 80px);max-height:224px;max-width:234px;position:absolute;right:0;top:40px;transform:translateX(30%);transition:transform .4s;transition-delay:.6s;width:100%;z-index:1}}@media screen and (min-width:980px)and (min-width:1240px){.case-study-card:before{transform:translateX(10%)}}@media screen and (min-width:980px){.case-study-card.is-open:before{transform:translateX(60%);transition-delay:0s}}@media screen and (min-width:980px){.case-study-card-wrap{align-items:stretch;display:flex;flex-shrink:0;min-height:304px;position:relative;transition:width .4s;transition-delay:.6s;width:calc(200% + 42px);z-index:2}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-wrap{transition-delay:0s;width:100%}}@media screen and (min-width:980px){.case-study-card-body{display:flex;flex-direction:column;padding-right:80px!important}.case-study-card-body>.row{align-self:stretch}}@media screen and (min-width:980px){.case-study-card-toggle{background:#fff;box-shadow:0 8px 20px rgba(108,117,125,.2);border-radius:100%;cursor:pointer;height:56px;position:relative;width:56px}.case-study-card-toggle:after,.case-study-card-toggle:before{background:#257af4;content:"";display:block;height:4px;left:calc(50% - 15px);position:absolute;top:calc(50% - 2px);transition:opacity .2s,transform .2s;width:30px}.case-study-card-toggle:after{transform:rotate(90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:before{opacity:0;transform:rotate(-90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:after{transform:rotate(0)}}@media screen and (min-width:980px){.case-study-card .col-lg-3,.case-study-card .col-lg-auto{opacity:0;transform:translateX(24px);transition:opacity .4s,transform .4s}}@media screen and (min-width:980px){.case-study-card .col-lg-3{transition-delay:0s}}@media screen and (min-width:980px){.case-study-card .col-lg-auto{transition-delay:.2s}}@media screen and (min-width:980px)and (min-width:980px){.case-study-card .col-lg-auto{max-width:605px;width:calc(100% - 319px)}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3,.case-study-card.is-open .col-lg-auto{opacity:1;transform:none}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3{transition-delay:.4s}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-auto{transition-delay:.2s}}.footer-copy{white-space:nowrap}form .form-control{border:1px solid #6c757d;border-radius:6px;height:auto;line-height:20px;min-height:44px;padding:12px 16px;width:100%}form .form-control,form .form-control:focus{box-shadow:0 8px 20px rgba(108,117,125,.2);color:#212529}form .form-control:focus{border-color:#212529}form .form-control::-moz-placeholder{color:#6c757d}form .form-control:-ms-input-placeholder{color:#6c757d}form .form-control::placeholder{color:#6c757d}form select.form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}form select.form-control:not([data-chosen]){color:#6c757d}form .btn-secondary:active,form .btn-secondary:hover{color:#212529;background:#fc0;border-color:#fc0}.hero{overflow:visible;position:relative}.hero,.hero-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.hero-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.hero>.container{position:relative;z-index:2}.hero.has-offset{margin-bottom:-160px;padding-bottom:160px}.base-hero{height:22.5vw;max-height:324px;min-height:280px}.index-hero{background-image:url(/images/backgrounds/bg-hero-home.svg);height:68vw;max-height:980px}.index-hero,.other-hero{max-width:2448px;width:170vw}.other-hero{background-image:url(/images/backgrounds/bg-hero.svg)}.bg-footer-cta{background-image:url(/images/backgrounds/bg-footer-cta.svg);width:2448px}.quickstart-bg{background-image:url(/images/backgrounds/bg-quick-start.svg);height:40vw;top:220px;width:170vw}hr{background:#f1f6f9;border:0;display:block;height:4px;margin:0;width:100%}hr.is-small{height:2px}hr.is-large{height:8px}hr.is-medium{background:#d6dbdf}hr.is-dark{background:#495057}hr.is-yellow{background:linear-gradient(90deg,#ff8c00,#ff8c00 8px,#fc0 16px,rgba(255,204,0,0));-webkit-clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);height:8px}.icon{display:block;height:48px;margin-bottom:24px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center}@media screen and (min-width:576px){.icon{height:64px}}@media screen and (min-width:980px){.icon{height:80px}}img{max-width:100%}.kicker{color:#6c757d;font-family:Hind Siliguri,sans-serif;font-size:.875rem;font-weight:600;letter-spacing:1px;margin:0}@media screen and (max-width:978.98px){.lead{font-size:1.125rem}}.navbar-clickhouse{border-bottom:4px solid #f1f6f9;height:142px}.navbar-clickhouse>.container{flex-wrap:wrap}.navbar-super{flex-shrink:0;width:100%}.navbar-super ul{list-style:none}.navbar-super li:not(:last-child){margin-bottom:0;margin-right:24px}.navbar-super a{align-items:center;color:#212529;display:flex;font-size:.875rem}.navbar-super a:active,.navbar-super a:hover{color:#257af4;text-decoration:none}.navbar-super img{flex-shrink:0;margin-right:4px}.navbar-brand-clickhouse{background:no-repeat url(../images/logo-clickhouse.svg);background-size:contain;flex-shrink:0;height:28px;margin-right:48px;padding:0;width:180px}.navbar-nav{align-items:center;height:46px}.navbar .nav-item:not(:last-child){margin-bottom:0;margin-right:24px}.navbar .nav-link{color:#212529}.navbar .nav-link:active,.navbar .nav-link:hover{color:#257af4}.navbar .navbar-nav{flex-direction:row}@media screen and (max-width:978.98px){.navbar>.container{padding-left:20px;padding-right:20px}.navbar .navbar-toggler{height:24px;padding:0;width:24px}.navbar .navbar-toggler:focus{outline:none}.navbar .navbar-toggler-icon{background:no-repeat url(../images/icons/icon-menu.svg);background-position:50%;background-size:contain;height:24px;width:24px}.navbar .navbar-collapse{background:#fff;border-bottom:4px solid #f1f6f9;height:56px;left:0;padding:0 20px 16px;position:absolute;right:0;top:100%}.navbar .nav-link{font-size:.875rem;white-space:nowrap}}@media screen and (max-width:615.98px){.navbar .navbar-collapse{height:auto}.navbar .navbar-nav{flex-direction:column;height:auto}.navbar .nav-item:not(:last-child){margin-bottom:16px;margin-right:0}}@media screen and (max-width:399.98px){.navbar{height:80px}}.page,.photo-frame{overflow:hidden;width:100%}.photo-frame{background:hsla(0,0%,100%,.6);border-radius:100%;box-shadow:0 8px 20px rgba(108,117,125,.2);display:block;margin-bottom:24px;max-width:160px;position:relative}.photo-frame:before{content:"";display:block;padding-bottom:100%;width:100%}.photo-frame img{display:block;height:100%;left:0;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;position:absolute;top:0;width:100%}.pullquote{position:relative;width:70%}.pullquote:before{background:no-repeat url(../images/backgrounds/bg-quotes.svg);background-position:50%;background-size:100%;content:"";mix-blend-mode:multiply;right:56px;width:calc(100% - 16px);z-index:2}.pullquote-bg,.pullquote:before{bottom:0;display:block;position:absolute;top:0}.pullquote-bg{right:0;width:calc(50vw + 28.57143%);z-index:1}.pullquote-body{padding:64px 40px 64px 0;position:relative;z-index:3}.pullquote-quote{font-family:Hind Siliguri,sans-serif;font-size:32px;font-weight:700}.pullquote-citation{font-size:1.125rem}.section{overflow:visible;position:relative}.section,.section-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.section-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.section>.container{position:relative;z-index:2}.social-icons{align-items:center;display:flex}.social-icons>a{aspect-ratio:24/24;background:#6c757d;display:block;height:24px;width:24px;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background .2s}.social-icons>a:active,.social-icons>a:hover{background:#212529}.social-icons>a+a{margin-left:32px}.social-icons-facebook{-webkit-mask-image:url(/images/icons/icon-facebook-gray.svg);mask-image:url(/images/icons/icon-facebook-gray.svg)}.social-icons-twitter{-webkit-mask-image:url(/images/icons/icon-twitter-gray.svg);mask-image:url(/images/icons/icon-twitter-gray.svg);width:31px}.social-icons-linkedin{-webkit-mask-image:url(/images/icons/icon-linkedin-gray.svg);mask-image:url(/images/icons/icon-linkedin-gray.svg)}.social-icons-linkedin-alt{-webkit-mask-image:url(/images/icons/icon-linkedin-alt-gray.svg);mask-image:url(/images/icons/icon-linkedin-alt-gray.svg)}.social-icons.size-small>a{height:20px;width:20px}.social-icons.size-small>a:active,.social-icons.size-small>a:hover{background:#212529}.social-icons.size-small>a+a{margin-left:16px}.tabs{position:relative}.tabs:before{background:#fff;border-radius:7px 7px 0 0;content:"";display:block;height:8px;left:1px;position:absolute;right:1px;top:68px;z-index:10}@media screen and (min-width:1240px){.tabs:before{top:76px}}.tabs-body{background:#fff;border-radius:8px;border:1px solid #6c757d;box-shadow:0 8px 20px rgba(108,117,125,.2);padding:24px}@media screen and (min-width:980px){.tabs-body{padding:32px}}@media screen and (min-width:1240px){.tabs-body{padding:40px}}.tabs .nav-tabs{border-bottom:0;flex-wrap:nowrap;height:76px;margin:-20px -20px -9px;-webkit-mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);overflow:scroll;overflow-x:scroll;overflow-y:visible;padding:20px 20px 0;position:relative}@media screen and (min-width:940px){.tabs .nav-tabs{overflow:visible}}@media screen and (min-width:1240px){.tabs .nav-tabs{height:84px}}.tabs .nav-link{align-items:center;border-bottom:0;color:#6c757d;display:flex;font-size:.875rem;flex-shrink:0;height:56px;justify-content:center;padding:0 12px 8px;text-align:center;white-space:nowrap}@media screen and (min-width:1240px){.tabs .nav-link{height:64px;padding:0 16px 8px}}.tabs .nav-link.active{background:#fff;box-shadow:0 -4px 8px rgba(108,117,125,.1);font-weight:700;padding:0 16px 8px}@media screen and (min-width:980px){.tabs .nav-link.active{padding:0 24px 8px}}@media screen and (min-width:1240px){.tabs .nav-link.active{padding:0 32px 8px}}.tab-pane pre{background:#212529;border-radius:16px;color:#fff;padding:24px 16px}@media screen and (min-width:1240px){.tab-pane pre{padding:32px 24px}}.trailing-link{align-items:center;color:#212529;display:flex;font-size:.875rem;font-weight:700}.trailing-link:after{background:no-repeat url(../images/icons/icon-arrow.svg);background-position:100%;background-size:contain;content:"";display:block;height:12px;transition:transform .2s;width:20px}.trailing-link:active,.trailing-link:hover{color:#212529;text-decoration:none}.trailing-link:active:after,.trailing-link:hover:after{transform:translateX(8px)}.trailing-link.span-full:after{margin-left:auto}ul{color:#495057;list-style-type:square;padding-left:1.25em}ul li:not(:last-child){margin-bottom:16px}ul li::marker{color:#ff3939}ul.has-separators{list-style:none;padding:0}ul.has-separators li:not(:last-child){border-bottom:4px solid #f1f6f9;margin-bottom:24px;padding-bottom:24px}.bg-gradient-secondary{background-image:linear-gradient(58deg,#ff6443 3%,#fe561d 24%,#e32f0d 93%)}.bg-gradient-light-orange{background-image:linear-gradient(90deg,rgba(255,203,128,0),#ffcb80)}.bg-offset-right{bottom:0;left:-24px;position:absolute;top:0;width:calc(100vw + 24px);z-index:-1}@media screen and (min-width:1240px){.bg-offset-right{left:-96px;width:calc(100vw + 96px)}}.bg-inset-right{bottom:0;left:40px;position:absolute;top:0;width:calc(100vw - 40px);z-index:-1}@media screen and (min-width:980px){.bg-inset-right{left:96px;width:calc(100vw - 96px)}}.has-border-left{border-left:8px solid #f1f6f9;padding-left:16px}.font-xl{font-size:1.25rem}.font-lg{font-size:1.125rem}.font-sm{font-size:.875rem}.font-xs{font-size:.625rem}.font-weight-semibold{font-weight:600}.display-5{color:#212529;font-size:20px;font-weight:500}.display-6{color:#212529;font-size:14px;font-weight:700}.text-decoration-underline{text-decoration:underline}.text-upper{text-transform:uppercase} \ No newline at end of file +@media screen and (max-width:978.98px){.btn{padding:8px 16px}}@media screen and (max-width:978.98px){.btn-lg{padding:12px 24px}}.btn-primary,.btn-primary:active,.btn-primary:hover{color:#212529}.btn-outline-primary{background:#fffaf0;border-color:#fc0;color:#212529}.btn-outline-primary:active,.btn-outline-primary:hover{background:#fc0;border-color:#fc0;color:#212529}.btn-secondary{border-color:#212529;color:#fff}.btn-outline-secondary,.btn-secondary:active,.btn-secondary:hover{background:#fff;border-color:#212529;color:#212529}.btn-outline-secondary:active,.btn-outline-secondary:hover{background:#212529;border-color:#212529;color:#fff}.btn-tertiary{border-color:#257af4;color:#fff}.btn-tertiary:active,.btn-tertiary:hover{background:#257af4;border-color:#257af4;color:#fff}.btn-outline-tertiary{background:#e3f1fe;color:#257af4}.btn-outline-tertiary:active,.btn-outline-tertiary:hover{background:#257af4;color:#fff}.btns{align-items:center;display:flex;justify-content:center}.btns .btn+.btn{margin-left:24px}.btns .btn-lg+.btn-lg{margin-left:40px}.card{box-shadow:0 8px 20px rgba(108,117,125,.2);overflow:hidden;transition:box-shadow .2s,transform .2s;width:100%}.card,.card-body{position:relative}.card-body{z-index:10}.card.is-large .card-body{padding:40px}.card.bg-primary-light{border-color:#fc0}.card.has-dark-border{border-color:#6c757d}.card.has-pattern:after,.card.has-pattern:before{background-repeat:no-repeat;background-size:auto 100%;bottom:0;content:"";display:block;position:absolute;top:0;width:72px}.card.has-pattern:before{background-image:url(../images/backgrounds/bg-card-pattern-blue-1.png);background-position:0 0;left:0}.card.has-pattern:after{background-image:url(../images/backgrounds/bg-card-pattern-blue-2.png);background-position:100% 0;right:0}.card.has-hover:active,.card.has-hover:hover,a.card:active,a.card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2);transform:translateY(-8px)}.card.has-highlight:after,.card.has-hover:after,a.card:after{content:"";display:block;height:8px;margin-top:auto;transition:background .2s;width:100%}.card.has-highlight:after,.card.has-hover:active:after,.card.has-hover:hover:after,a.card:active:after,a.card:hover:after{background:#e3f1fe}.case-study-cards{-moz-column-gap:40px;column-gap:40px;display:grid;grid-template-columns:1fr;row-gap:40px;padding-bottom:40px;position:relative}.case-study-cards>div{align-items:stretch;display:flex}.case-study-cards:before{background:#d6dbdf;bottom:0;content:"";display:block;left:20px;position:absolute;top:40px;width:100vw}@media screen and (min-width:980px){.case-study-cards{grid-template-columns:repeat(2,minmax(0,1fr));row-gap:80px;padding-bottom:120px}.case-study-cards:before{left:-40px;top:120px}}.case-study-card{align-items:stretch;flex-direction:row;flex-shrink:0;left:0;transition:box-shadow .2s,left .4s,width .4s,z-index 0s;transition-delay:0s,.6s,.6s,0s;width:100%;z-index:2}@media screen and (max-width:979.98px){.case-study-card .row{min-height:0!important}}@media screen and (min-width:980px){.case-study-card:active,.case-study-card:hover{box-shadow:0 12px 32px rgba(108,117,125,.2)}.case-study-card:not(.is-open){cursor:pointer}.case-study-card.is-open{transform:none!important;transition-delay:0s,0s,0s,0s;width:calc(200% + 40px);z-index:10}.case-study-card.is-closing{z-index:10}.case-study-card.open-left.is-open{left:calc(-100% - 40px)}.case-study-card:before{background:no-repeat url(../images/backgrounds/bg-card-pattern-red.png);background-position:100%;background-size:contain;content:"";display:block;height:calc(100% - 80px);max-height:224px;max-width:234px;position:absolute;right:0;top:40px;transform:translateX(30%);transition:transform .4s;transition-delay:.6s;width:100%;z-index:1}}@media screen and (min-width:980px)and (min-width:1240px){.case-study-card:before{transform:translateX(10%)}}@media screen and (min-width:980px){.case-study-card.is-open:before{transform:translateX(60%);transition-delay:0s}}@media screen and (min-width:980px){.case-study-card-wrap{align-items:stretch;display:flex;flex-shrink:0;min-height:304px;position:relative;transition:width .4s;transition-delay:.6s;width:calc(200% + 42px);z-index:2}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-wrap{transition-delay:0s;width:100%}}@media screen and (min-width:980px){.case-study-card-body{display:flex;flex-direction:column;padding-right:80px!important}.case-study-card-body>.row{align-self:stretch}}@media screen and (min-width:980px){.case-study-card-toggle{background:#fff;box-shadow:0 8px 20px rgba(108,117,125,.2);border-radius:100%;cursor:pointer;height:56px;position:relative;width:56px}.case-study-card-toggle:after,.case-study-card-toggle:before{background:#257af4;content:"";display:block;height:4px;left:calc(50% - 15px);position:absolute;top:calc(50% - 2px);transition:opacity .2s,transform .2s;width:30px}.case-study-card-toggle:after{transform:rotate(90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:before{opacity:0;transform:rotate(-90deg)}}@media screen and (min-width:980px){.case-study-card.is-open .case-study-card-toggle:after{transform:rotate(0)}}@media screen and (min-width:980px){.case-study-card .col-lg-3,.case-study-card .col-lg-auto{opacity:0;transform:translateX(24px);transition:opacity .4s,transform .4s}}@media screen and (min-width:980px){.case-study-card .col-lg-3{transition-delay:0s}}@media screen and (min-width:980px){.case-study-card .col-lg-auto{transition-delay:.2s}}@media screen and (min-width:980px)and (min-width:980px){.case-study-card .col-lg-auto{max-width:605px;width:calc(100% - 319px)}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3,.case-study-card.is-open .col-lg-auto{opacity:1;transform:none}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-3{transition-delay:.4s}}@media screen and (min-width:980px){.case-study-card.is-open .col-lg-auto{transition-delay:.2s}}.footer-copy{white-space:nowrap}form .form-control{border:1px solid #6c757d;border-radius:6px;height:auto;line-height:20px;min-height:44px;padding:12px 16px;width:100%}form .form-control,form .form-control:focus{box-shadow:0 8px 20px rgba(108,117,125,.2);color:#212529}form .form-control:focus{border-color:#212529}form .form-control::-moz-placeholder{color:#6c757d}form .form-control:-ms-input-placeholder{color:#6c757d}form .form-control::placeholder{color:#6c757d}form select.form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}form select.form-control:not([data-chosen]){color:#6c757d}form .btn-secondary:active,form .btn-secondary:hover{color:#212529;background:#fc0;border-color:#fc0}.hero{overflow:visible;position:relative}.hero,.hero-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.hero-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.hero>.container{position:relative;z-index:2}.hero.has-offset{margin-bottom:-160px;padding-bottom:160px}.base-hero{height:22.5vw;max-height:324px;min-height:280px}.index-hero{background-image:url(/images/backgrounds/bg-hero-home.svg);height:68vw;max-height:980px}.index-hero,.other-hero{max-width:2448px;width:170vw}.other-hero{background-image:url(/images/backgrounds/bg-hero.svg)}.bg-footer-cta{background-image:url(/images/backgrounds/bg-footer-cta.svg);width:2448px}.quickstart-bg{background-image:url(/images/backgrounds/bg-quick-start.svg);height:40vw;top:220px;width:170vw}hr{background:#f1f6f9;border:0;display:block;height:4px;margin:0;width:100%}hr.is-small{height:2px}hr.is-large{height:8px}hr.is-medium{background:#d6dbdf}hr.is-dark{background:#495057}hr.is-yellow{background:linear-gradient(90deg,#ff8c00,#ff8c00 8px,#fc0 16px,rgba(255,204,0,0));-webkit-clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);clip-path:polygon(8px 100%,0 100%,0 0,8px 0,8px 100%,16px 100%,16px 0,100% 0,100% 100%);height:8px}.icon{display:block;height:48px;margin-bottom:24px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center}@media screen and (min-width:576px){.icon{height:64px}}@media screen and (min-width:980px){.icon{height:80px}}img{max-width:100%}.kicker{color:#6c757d;font-family:Hind Siliguri,sans-serif;font-size:.875rem;font-weight:600;letter-spacing:1px;margin:0}@media screen and (max-width:978.98px){.lead{font-size:1.125rem}}.logo{display:block;height:36px;max-width:220px;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;width:100%}.navbar-clickhouse{border-bottom:4px solid #f1f6f9;height:142px}.navbar-clickhouse>.container{flex-wrap:wrap}.navbar-super{flex-shrink:0;width:100%}.navbar-super ul{list-style:none}.navbar-super li:not(:last-child){margin-bottom:0;margin-right:24px}.navbar-super a{align-items:center;color:#212529;display:flex;font-size:.875rem}.navbar-super a:active,.navbar-super a:hover{color:#257af4;text-decoration:none}.navbar-super img{flex-shrink:0;margin-right:4px}.navbar-brand-clickhouse{background:no-repeat url(../images/logo-clickhouse.svg);background-size:contain;flex-shrink:0;height:28px;margin-right:48px;padding:0;width:180px}.navbar-nav{align-items:center;height:46px}.navbar .nav-item:not(:last-child){margin-bottom:0;margin-right:24px}.navbar .nav-link{color:#212529}.navbar .nav-link:active,.navbar .nav-link:hover{color:#257af4}.navbar .navbar-nav{flex-direction:row}@media screen and (max-width:978.98px){.navbar>.container{padding-left:20px;padding-right:20px}.navbar .navbar-toggler{height:24px;padding:0;width:24px}.navbar .navbar-toggler:focus{outline:none}.navbar .navbar-toggler-icon{background:no-repeat url(../images/icons/icon-menu.svg);background-position:50%;background-size:contain;height:24px;width:24px}.navbar .navbar-collapse{background:#fff;border-bottom:4px solid #f1f6f9;height:56px;left:0;padding:0 20px 16px;position:absolute;right:0;top:100%}.navbar .nav-link{font-size:.875rem;white-space:nowrap}}@media screen and (max-width:615.98px){.navbar .navbar-collapse{height:auto}.navbar .navbar-nav{flex-direction:column;height:auto}.navbar .nav-item:not(:last-child){margin-bottom:16px;margin-right:0}}@media screen and (max-width:399.98px){.navbar{height:80px}}.page,.photo-frame{overflow:hidden;width:100%}.photo-frame{background:hsla(0,0%,100%,.6);border-radius:100%;box-shadow:0 8px 20px rgba(108,117,125,.2);display:block;margin-bottom:24px;max-width:160px;position:relative}.photo-frame:before{content:"";display:block;padding-bottom:100%;width:100%}.photo-frame img{display:block;height:100%;left:0;-o-object-fit:contain;object-fit:contain;-o-object-position:center;object-position:center;position:absolute;top:0;width:100%}.pullquote{position:relative;width:70%}.pullquote:before{background:no-repeat url(../images/backgrounds/bg-quotes.svg);background-position:50%;background-size:100%;content:"";mix-blend-mode:multiply;right:56px;width:calc(100% - 16px);z-index:2}.pullquote-bg,.pullquote:before{bottom:0;display:block;position:absolute;top:0}.pullquote-bg{right:0;width:calc(50vw + 28.57143%);z-index:1}.pullquote-body{padding:64px 40px 64px 0;position:relative;z-index:3}.pullquote-quote{font-family:Hind Siliguri,sans-serif;font-size:32px;font-weight:700}.pullquote-citation{font-size:1.125rem}.section{overflow:visible;position:relative}.section,.section-bg{background-repeat:no-repeat;background-position:50%;background-size:cover}.section-bg{display:block;height:100%;left:50%;position:absolute;top:0;transform:translateX(-50%);z-index:1}.section>.container{position:relative;z-index:2}.social-icons{align-items:center;display:flex}.social-icons>a{aspect-ratio:24/24;background:#6c757d;display:block;height:24px;width:24px;-webkit-mask-position:center;mask-position:center;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;transition:background .2s}.social-icons>a:active,.social-icons>a:hover{background:#212529}.social-icons>a+a{margin-left:32px}.social-icons-facebook{-webkit-mask-image:url(/images/icons/icon-facebook-gray.svg);mask-image:url(/images/icons/icon-facebook-gray.svg)}.social-icons-twitter{-webkit-mask-image:url(/images/icons/icon-twitter-gray.svg);mask-image:url(/images/icons/icon-twitter-gray.svg);width:31px}.social-icons-linkedin{-webkit-mask-image:url(/images/icons/icon-linkedin-gray.svg);mask-image:url(/images/icons/icon-linkedin-gray.svg)}.social-icons-linkedin-alt{-webkit-mask-image:url(/images/icons/icon-linkedin-alt-gray.svg);mask-image:url(/images/icons/icon-linkedin-alt-gray.svg)}.social-icons.size-small>a{height:20px;width:20px}.social-icons.size-small>a:active,.social-icons.size-small>a:hover{background:#212529}.social-icons.size-small>a+a{margin-left:16px}.tabs{position:relative}.tabs:before{background:#fff;border-radius:7px 7px 0 0;content:"";display:block;height:8px;left:1px;position:absolute;right:1px;top:68px;z-index:10}@media screen and (min-width:1240px){.tabs:before{top:76px}}.tabs-body{background:#fff;border-radius:8px;border:1px solid #6c757d;box-shadow:0 8px 20px rgba(108,117,125,.2);padding:24px}@media screen and (min-width:980px){.tabs-body{padding:32px}}@media screen and (min-width:1240px){.tabs-body{padding:40px}}.tabs .nav-tabs{border-bottom:0;flex-wrap:nowrap;height:76px;margin:-20px -20px -9px;-webkit-mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);mask-image:linear-gradient(90deg,transparent,#000 20px,#000 calc(100% - 20px),transparent);overflow:scroll;overflow-x:scroll;overflow-y:visible;padding:20px 20px 0;position:relative}@media screen and (min-width:940px){.tabs .nav-tabs{overflow:visible}}@media screen and (min-width:1240px){.tabs .nav-tabs{height:84px}}.tabs .nav-link{align-items:center;border-bottom:0;color:#6c757d;display:flex;font-size:.875rem;flex-shrink:0;height:56px;justify-content:center;padding:0 12px 8px;text-align:center;white-space:nowrap}@media screen and (min-width:1240px){.tabs .nav-link{height:64px;padding:0 16px 8px}}.tabs .nav-link.active{background:#fff;box-shadow:0 -4px 8px rgba(108,117,125,.1);font-weight:700;padding:0 16px 8px}@media screen and (min-width:980px){.tabs .nav-link.active{padding:0 24px 8px}}@media screen and (min-width:1240px){.tabs .nav-link.active{padding:0 32px 8px}}.tab-pane pre{background:#212529;border-radius:16px;color:#fff;padding:24px 16px}@media screen and (min-width:1240px){.tab-pane pre{padding:32px 24px}}.trailing-link{align-items:center;color:#212529;display:flex;font-size:.875rem;font-weight:700}.trailing-link:after{background:no-repeat url(../images/icons/icon-arrow.svg);background-position:100%;background-size:contain;content:"";display:block;height:12px;transition:transform .2s;width:20px}.trailing-link:active,.trailing-link:hover{color:#212529;text-decoration:none}.trailing-link:active:after,.trailing-link:hover:after{transform:translateX(8px)}.trailing-link.span-full:after{margin-left:auto}ul{color:#495057;list-style-type:square;padding-left:1.25em}ul li:not(:last-child){margin-bottom:16px}ul li::marker{color:#ff3939}ul.has-separators{list-style:none;padding:0}ul.has-separators li:not(:last-child){border-bottom:4px solid #f1f6f9;margin-bottom:24px;padding-bottom:24px}.bg-gradient-secondary{background-image:linear-gradient(58deg,#ff6443 3%,#fe561d 24%,#e32f0d 93%)}.bg-gradient-light-orange{background-image:linear-gradient(90deg,rgba(255,203,128,0),#ffcb80)}.bg-offset-right{bottom:0;left:-24px;position:absolute;top:0;width:calc(100vw + 24px);z-index:-1}@media screen and (min-width:1240px){.bg-offset-right{left:-96px;width:calc(100vw + 96px)}}.bg-inset-right{bottom:0;left:40px;position:absolute;top:0;width:calc(100vw - 40px);z-index:-1}@media screen and (min-width:980px){.bg-inset-right{left:96px;width:calc(100vw - 96px)}}.has-border-left{border-left:8px solid #f1f6f9;padding-left:16px}.font-xl{font-size:1.25rem}.font-lg{font-size:1.125rem}.font-sm{font-size:.875rem}.font-xs{font-size:.625rem}.font-weight-semibold{font-weight:600}.display-5{color:#212529;font-size:20px;font-weight:500}.display-6{color:#212529;font-size:14px;font-weight:700}.text-decoration-underline{text-decoration:underline}.text-upper{text-transform:uppercase} \ No newline at end of file diff --git a/website/images/logos/logo-almaz-capital.svg b/website/images/logos/logo-almaz-capital.svg new file mode 100644 index 00000000000..ed2357083ed --- /dev/null +++ b/website/images/logos/logo-almaz-capital.svg @@ -0,0 +1 @@ +AC Logo \ No newline at end of file diff --git a/website/images/logos/logo-altimeter-capital.png b/website/images/logos/logo-altimeter-capital.png new file mode 100644 index 0000000000000000000000000000000000000000..7c0d3b25b8d90a3b65a4ba792eac46ad88a12c3c GIT binary patch literal 3600 zcmYk9byySr_Qy9Sl7k71?oxVygdoz5NQ@2#QYzhy8a3$@7%7YrkoXcJFmQl$jFys= z5F}MlLHKd6_xIfUJm)#*b>5%%InVj`6K9~W2_|PI2LJ$IZ7nq;006jp&1h24b(R>f zgkP6v13kF+8X{wAGa1feT-kK6zKc=!jR7j11`@P&&1LUb}n@ zyE>D%3S3y>@;Ohv=ne2NP4x#OM_|_^Y+<7@=Gg1B7J@Ni1+Ox5^H#IQcHR)2bk-+8kUw znw$ZNtEuB1tR5-3apnOr!w|IipI6u$j7p3d#WfaGUCU!%u70UbXKl@xo3X&~r*Ab^ zGNx}yE)R^SZm3{Rcg`bCG=>QM2fMxb>@P0oR*zz8{bnY(t_}87~5G=zv>>KvUpWl6ZEe zy3>fE*lCuc2EL7EG{8qWWwqG;K~J5OhNF7QF%{~e;)RYC0Eg8Kl2NbuChyhM1HSz38Rrk-P1-MRu6|Gi?>aos8jOH1~)*H7?bG%Wbay8q#5~-7f(T-U66X2ZFNseuzC#UiI2VPNt&Y=9AJTsAqNHlGZlzJHq~Qow%ai2=haI`gkV5AUL2MR`aH z8hQmICGRC-?H}j7YSW+Jm?UV0p-JM9zMo{=Ej~A314?B)m772Zc|y6Z=bWUc?kW=U zdlWK|H@dQuIiG!(I#yD{lVPM^1#F<1{swD2+Oiep?_SOCwOhV_Kk>VmEkwarc^g^z zW4kW$Vgth`Q=Pz63E?gq$~BxH2g^Lzqg`?nP;h?|*$GA|x-;0sL^|~S6qG&pSlY*N zG4aYZR2I|LhEE#$`tfCbGHcrM|9&3OYcnoVAk>m7*JE23j(%Y9kCWlm2zQ>&EPF(| z8DY3(w^AMrfZ`sY@6w5XM0IU$5p8JBQ1caOuvmkV8=-+Ro< zvPg*!E8#D$5}Zvxt1$3^%eG6dqB*w|g4p2s8!9xP?$>_iXLq+5JydegfM{&j{9Hox zSLm{kUM9_+LyUgE$sXYFu&NSyONr`({|LNe+pXyCn~UHNN@WFk?*^V@i(DOCS}2t4a!ty2>MJ}W{NxF#*!Yz~ag)Vtxc z?fRL2&m!qBj5CmsCad9$GzMfm>>lU(#n(oJ!Xl*MZ z|FVs~oEis6hokJ{?bm{AV2~}>h5Tb!y4m*BlvwfgiePDn#j_`T@>Xv{Wpx@UirO>p2K3Fn|6!osFaf& zOmg~+ROK6WV-3gGkMWDfF2tNGgj?$c(L~)%Y(RntTzay>RYh@{I3<@KKL1g4_3_t& z`7A69g5Dq+z((R@-@3+=n;c$+Ng_%Z6SSfvZ=Ze-pV#CnNTJKL#FFXk?-+|T(q1e< z91H}J)Ohzz$*SmkNaeSR*eR&!%b_w5Yk1!h9WtiSpsvNZ?5zjT99ZvlNFUjD1;! zMf^oHN&e4elg4^X2i1{x0Y0;n5!0OD`uD>$3^7{bG^^FUjRi}avVVm9MBK~V)e3d6=qodJ= zqb5rV0^9|!`)Rht+Doc#a8*fD59iR@Zw^~NcX!W*9Lv94)#;ATY1S*1USK0R4j7cKoML6ZYu z82z(ORUE`?6-^SxES}B+(~U6yJ__NtRHf;Pt(65E4rY)Z(ffzgFuQzg$nGs^eaQ+F zQfi84#XA^qGls{>>l9Bbv*Rz1#~k=Z2mk2`lMQD`RPx4G0rw-Q=)r;srkV!#g|a&h z(r(rx4}D#kM{6k___akFu?zwmDT)1-e-&ZniF-GR{F+D!lK@-K55BlQ<85;ow%(OX z9wCj^nIdNUB3t!`w(H&g^o_*iyC^g3-%3#nRMkBBN0;W621vZ$`+D&b#ays#-pX$3 zb3v&z;v-D>+=Jb~su*1f8ksDPBWFKi_fRa_S1CN4cL44=fz1#`LBsBxFPQl&6*GT1 zzPO2+CtDIsqhekZH9}(jVS*)^x^l&bIPj7RWkP~FXqi0cNZ*W) zWW`n6>k!SM^Gh9bG2;y0D?nHtlM>+@6#!QlhX~tGh+^{wBy)z2yqF2*^JXt;0gT{y z;02kUeI5FW%7%kp)%R1!C+;+Pee+Nz1 z;+Vs`OwOBYR}b;bC`|_b=$cErS#Ukag%n;{20tQp)L5RdC*O2_Dx_S{5Tis-KeI1Vtxpy%O)ams$)G@UDXcdTHST>PUIlq{Hv0ZY<*-pU z_jcZ?G25%9n8Zx9Fs;?-V1Ksh(K|lm!FUXrY5SLdECG2*X;$%)C$HG6sps5?IiS^E zPuZYdIZXEAi$a(B znVz-{n0thu%)@WfhvYv~U+Rp?^VF!n zJ=Z+#z?!1k-(3XyA|8E)mhZJWtr^aqf7T##oZp!#s@SZzS=Ml#9!N-xqbnS$3wZ~m z6%<=~q01m}sF&j;1r^d5T4^7n10>q;%>HA4)eO%J6&r1GU8|Jx+SsPNJgjM2pt#ss ze5OF$G@laI7~s)P)&m2eZAkmO=5B}(OPdSOeLw%R;sLZpL`f$JhM%>@KkN0U4KS-b zB@0+s=+f5<$b8oGM$Rn|U4uEujVU0{6FWQqW2q-g(p$s(l4#oKut=h#Sq;Pl+^&VD z*qbd={IX+3bz+Jpdpq)=-PT>?LS2q{>CCNAkTJ#h8kD)zzE$5f(pT4?ncGYbMQ?8j z%a1%Qv*36(Iyjq8H&Cj*xv?!gmE0N0qU|F}?#@>(0*WbZ1!I7@aZd@<;DaN|!#m{< z`I6MX0Bs-Xwd*hs8B;8-$c}JqWb8?I367`TInt>#u@$H?#mB2s*D>paYHat_^`Qil+}uJG6uE=NIU^vI+EC((@KK zlxKq<`}e~y1rJSzMi(_kJc0w7B4MO4Kuj5hla+|IsPxoGSzyT3SpqMljnQUIb|v#n zaUcch`7JuKbiL)V#BxSq-*gQUw;=WbKNZFjfks+T{NfHNO1*z58v;*#UcFXAUSQof z9MDvfI%{%7&l64Z04F$#;2*%q-z_-yvkIP)YII7iq)#arzB^Rd)#jt>`K;JQ^C@&V zJZ}rCAfQ_>b@f8|YvN|3H*!*cbP35_Z|pWkp?E~UF9$>KU4N+n+UojhwJM0n{{!qn B!Xf|w literal 0 HcmV?d00001 diff --git a/website/images/logos/logo-coatue.png b/website/images/logos/logo-coatue.png new file mode 100644 index 0000000000000000000000000000000000000000..c9331141f5a6476339cb38b5c8209ab87b4ef8e8 GIT binary patch literal 6128 zcmdT|Wmgmo(LoxINa;y&aBwKp-zpp6;NYSE`LV?K|1^o?k@!DT zp{He}^3VQ%5fRaP<$qYkeGPOBac-C!F{B4FbOi5u5hK3O&i)hVc3k7gfodotCl)$p zdC+0;5;h_OwHBVPDSBo843!y01(4mX?`?0N;o#7;sw*oP`7a&iEq?kg|DXH_0aF|? zM^qTu294Kh<-6kDVaXsCHkq|=h!TL+~2rqJvpnNO6zK(Be7-Q*nP0n3IHYC>K!~ZeYa&0(>QWn zbaC!Fe#rn2dAQw_w*IQ&U*O`{08Km^Sv|Fy>|O9(ddVW|Z@q>acS`G{H^3{$xaws4 zXT*iLXXi$e$ia(84llSm9oQn1w%G?R*nHYMCFY4InU!0~F7t~l;QZ6$Imv6hnhTpG zZ6*B5t{O6!mfmg%&24w3EqCxQXC5iBI=#@T50oBsVqq-z9LDrluOlo;{)m}8P0QZg z{LtR_C(>MtC7u0X5^B9bDI1n+SB$K=2{6Oe#O)9d{KA3xZvJj{8pW9Thw@!%42&EP z=lzDq__yx*#HE=hgHRea`wMGKur@|D?(_F)epupg<6U2YN5(F3b89DIgYScZf^34H zskLz(M>e0pfQpfdY&iE;Xv84hgB5~7bjFSX=Bo$ovPhc#ZjyxF=GuB(z<|~DxCFvI z;!or8-_~s=wpeLX_(Pe1z@(A6kWrFV=<&%E(;Q^`7RW?yO`oJ;cgbW_IS9}Gol#G5pHK6G{BgPNc(bT&JE{^(#Nyh z_6rN({H|vXV!!%ZlHG*#n=kE~?rg4*Dq^&x5mgt+OolZgD~w%{GGA$W%2el2Ol(klz*J3>+1|oO?9*%J7Ne}=fWw+CZ{S=>W^oT}ms+Tqr2PSpcNf=sUJC=M1Xh^NbrE&p4q z3WOEBkxv-Ywg91}Z^-R*vikMPlbXG~tpOZaXFSsYhe5I+o9t+G2H7JNfT9K+CtNax zqkRC`^2X&(F*OAB>lYvy|aY@Y$92;Qu_=S=x|;JpL&w{lEbv^3{+Ne;C^vU8-D<{&XV1)$`NR7;Ah)EcQ|_D*Rd#q>g%&n8RN^tix=MaT46h+ z41EhaP2$qC4`<}{Tc-V{^+I5kcFbg;y%Gy?|W_+cYp@MN3x7a z&k@8<>c z2qSbV5R*yni(pyd5==NvYmXHLw%l!-WxF?Y?+n!GHbn2_&Ca{_C)-nU_@ji+I3J3 zw8;qrYZgnj9i7MgW*Qelc84rLeMaHRf4tsJnfgk4olSIt>%recX9LU<AG(f9!2;XTKcKyV;yPXT4~^LUNg-Z_+9Y<|OL&vC64%{TaZ zpx8t0U1L3kthc|utlsmpxw~e0lo)T4%7Obz7FsC3#kQZJ;`30xNdg7nSM`STT(Nom zh(McXRXLUigtFDoL$l^$VrG~UV8PKYQrAY4>@uL*gMwrQiYhT2Eqw9}*>bA8Oh zm^DV*3o((c5OKh4Gwa0|r=^hM*pmcU(GA<28E4K_O$Od)Sp| zN7kr~LiG+@T#_&jIbvKy6YP&#rcl>Ra%-+DFS)q-y^!9ps z$omog%U$y6yU=So6a_{~6&GRh@DW8d$Qq*ct70D^HtQjLW1JbYqe>jB@2Z=Z4(8yB@)ipbG2*h-qmd_5|Fp8sN!_nhYypPpn`O(QsrBxZNGb=1su%)d2fu(v19d<%XIi-+$BeU;Lz~w@K+(*ZH^aJI9RbC4iXuEVu`&(C(cE$ z2q+zv8B~%i8vlK_iUbU)+ai0`+C}>zt2m#0LY{QTeCcOawu$s8v&?LIw>}>SCPq5< z!xM=YK<`%8PH<85eaHB%W3bwTS|iC!0O$dZ8KT30zKKv&l|xJ`OFC_1s1EvcZwrY# zoRIiyK_UZfk6dV@D7PH%r0`Rcj7`2Kh`p1D^}-~8r~ZStL_k(l{9N6;S*wI47<}BT z_%3}SDigdgiP*(kDh9DoNsbvX-a(Q)^)D5nW9)3%QHIDrQvbqHqh2RZad#u^NRCXY zd3+rpcIWqtfakRxvn`??RISh@v4Hu~fOnI8OR{<;2?+(OO4?APdauf*Lq6*HC7 z0ciNw_}9InoUl2Xiu;JZDw|O_whvN}LV>2myG;tAc|<ro%Nfgv~8fT-R169MT z85|<8A;fY6#05DtPraYL@{*zSD6G& z4201v385Q;TLw<;Q%(?$)9u~2k8~$-&CZv8saSeVk2&YX=DLZ#d~J`c=xTZm|cEW|x zPmp)@O{BC)tW!7(K)v1$2>=Nrm-V*~#Mk)a9J!!E?*0F=B$0iykpotisKKseA6sMv zpE8YZpGkw{c|E3J{T>Evfezd~?}Lq03pyhY(d8?0oSVB{WFalPi)}L|j!x(htir`} zOU0LJaV^g$wA%4JF-uN4U-*YLPbAZx-kxMKvC3|H1z-c6S??5byJop(-7Q*HG=(hDOtCQ{c0(dmp6{FV}u<@TAg zF>Z$bLzQYUn;X;%0|iSrbhfG3D)uHAov4elag0g6sGYgvg5ar*!@)f=!?+z%^^yIh z+3w$C(vvHR+=E>S-1s3mbf93knwk%CLbeoGrYT0*T|WLdsFmzb6(sn! z;$dZa`tmAz%1BzIzQ_H8fGb!>cmIjj26NN#nc@j*p6c<6vs^|#)^X``@a)FAL* z=#vw5!OO~ut%+Ff1kPt@f(`WNfi{s`)!^UFS&^7{2j;V3HH7j1>fm_>5_O7q4d-7$XJ3jo_m0Nf!Agwaw9pnR@myFIjm^`$mqJ$| z_LU01XJDQJlo(Cz|1g?&RDNTOI7Hfbu}vgmnjUU%O5lM%n>fn;C4hvs*TzY94){{) z$U994u5|j4PE!(!g04pEKt{BQK9V-%hO_;={Y4q+4@->buE5q(2=p1cZyg=I5mX%f z9;%l0KuC4Wo$hGWDzh3i$QWexZ~LKZ_H@FlRLtm;TKxzeOcS=@vI`=g$yq6=N_REOUnAU6~q-fArTGSmf%cFN35iMyQ#dv7j@zL^o z2)yD!8V&f76{_PsCxIE#u+PRUgdVlUW|`bzJ*o6Vc$m<40j1Lcz2qJxaAX{r=L+s_ z5???#tM<=MtkT1wrPByQix%O3SFl#9f6v7Gy1L$Jw4uM5%lUyc+L~a1-KB&|Kc(9 zN2)Te(u1xd*&32z6u#C^OPv2Ut?O8gCYBpgB0;bh+TkLy{+^o4?q`V|rjI#}xs`~j zdYaXa-pfAt!36}*ZQUh2se&bdzON+X3EI8cji}^<9MVR}j9{?~=5l<(Xl&jAvy)_S zF*RJw5ES8-{bwRXwnnKCztoUXM!;vcoBq3d?4@QLmUOOYKwHw6gUBdCkCzOP$ERx zcvUE+q#Rf4nXu`E#REwpg4wU&F+_%66RGBKT&zFaP4D5+QJ~I1LvgRVSwRLrdFt zl+TweGWvRO_r+md%niL<#S>eq2D1!Wkdv>E(z&CS0Unb5)5uEBjWtZ=)p*kt<8~( z(7*JiR6%BV$ayntbpx*B!D2+pfW+;K^7SU%Wr}PO!$+oU_gOhsAH1A1Y2Ot~7;|#t zqAmL60+}isYZNn0*@|ig3gjngJ;esDWUP$%1kkUBr|3&b8VvD}Y68f5DaS*cv+aKJ zrmT~3Q_{c8v#C)kh+Qwsk4FER+ZXGEY#HV2rx0z()+KxE;)ZJ-g_^pBrb?K{l?JIW zf03M1P7$w=mON^k+W7kTm#Cn(A+xcFeh15s-N4oXvQ=+E>KoOovUv`wW`X)4!QM%k zk{XoHM5#9rmIVH=dz)W_G7?`j5iAj#$?R#QB4Z15gVheitjCkck_;%pl$>}B+J zjop%K{iViCb>3cU5+6X8E4z!+{8zM;D-PAONfsSAK1aQz!VA{jd;+VB{kaIcM6>(} zPb39L9cc6F?<8#y>u-}KRi9Pu=H&N+o1r#D!R(WPr>}ahpeEcQL{wLO{2NWG1%K?z zE+`^Eu`h)FYEy}Gv&!>dbGWRGOT;(svj*`nhP_Iy6+vrEHz<_#Nkz#Fi6!@Y|4IFqjwhvYTJ0EJG8JwwdaqgbUzDk^Q?-awoj|nVB^My$j znVAnE{jC~3ZFNnVr2go6y<#rU_hB$-h}<@8ez~rjTvzNK;PBhXk5)|Z;Aujy%cT_C z5P0E<=m{!7qAW{;nf6Of9+r z!rS|7n&>u+{v_SbM|({NA*YnPpQVf?ELmw|EoAhm__g`Bp3us#Yf=Yt$vm8|eztyd z*WeHlO?7gaTgUZ~;~j6)@QbtPR=8K1nsy_Q$kJ(2T`a%;MFlQj%wee@s&UtT>RM4_ zr=Qno>C8*JRq(#?XWy&y+V*)m!l%^v6n=#T;_08YUU6nw5U7H0zTt)vdyr6?>$h4^ zVH*G_0_xbmd(qyc&9VmEhNK;%Lm7+F3oE^!@hV?3o9%q;qoR3YdazuuusRE2oavKnNRb>fqSZUg_l8lJ~tZi0UT$@4D4ZCyWoqTAAV=j zb77Nny=|oi1}?9q$|#d(U)WfUu~Zk6{}^BXecO}Tt_NwcKCG>Hb{vU6JbmO*Fsp{4E~YX4a591(Lnn(+iLFd_aNp-U7oVQJ%EnM!{w@FUP%oG?ee{G@ftAqS?72o0#8YmlN_Lz!Ai28a@NJ(1ln2%WItm3rq3CvCo&rN5ZhT^U_)X zPp(Qnh)Dxwa=H~S+JucI_-rn*;P6qI<{AGHSdj<0nl@eJv0sv0yBWl#c=p3&Cdc)W z!tgKEAEHry2J4iQfmp&LK1Gd0tl||(o&!AhuU{BP3p-%!i|lnt3KmUfxr$dZJqeCT1O;C)ktO}p=}TJNrSZ)?xW|^ej7ooZ7o>g zXHOMIT=P{=2At==n&*WT<+;|F1SED|WIVD)1T&n6N9gWG8Fi=bJTF0N{T-Vh?0DrS zvg+hdYd z`0iZsEVIpWEYJ=H{HQL4tZ`UmuQtnwYT*1*KB4gHtWzWcq~eB(P|PqX)@ UHvtD5|NGmktLP|KDcXenAE`#v*8l(j literal 0 HcmV?d00001 diff --git a/website/images/logos/logo-firstmark.svg b/website/images/logos/logo-firstmark.svg new file mode 100644 index 00000000000..e529766bae4 --- /dev/null +++ b/website/images/logos/logo-firstmark.svg @@ -0,0 +1,579 @@ + + + logo-firstmark + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/website/images/logos/logo-leadedge-capital.png b/website/images/logos/logo-leadedge-capital.png new file mode 100644 index 0000000000000000000000000000000000000000..2c7671bfd67545d26216a8cbf78f886f434153b0 GIT binary patch literal 3029 zcmcguc{CJi8=q1l$(o(G6;YOCW-xX!#E_+=v5YJUBN@wd$5^sX5@U^QU34YJa>qJN zB&4x4QDe;5LYB-BAsWj!-|5`*-M_x?zwbHk`8~h$?B~4idC!BpVh0sGDsvP700>%J zU2+5f4zRd9m5+yu*`YTMaaHmaTW1Td&d<-+aCyKzzsN5vE-vPhzy6kA-inF}E-5Q3 z`)A~@+b5q_`r zUU6j-p;)qVB_v&IWwN#eQ5$ll_I>(f$d!g*BLo0&blCcmsWY0vVvdGB(UMB?s-NL` z=_T>zu{7uW`rb<2ZyqNPn)ZvE`oT47z8R^wktAUS6hf`k@KxH~rOz*F`Z`1O1nhu)z-a^WBzL{}7i& z6Lohke%d&lOeO${8=ZrGo^Fug)@#XvFNG}iKO?YF`Xz=Zo#6^_GVmy|lfVd`2n}-n zc#1yg9G!kDtSwO}WURMsZ#2(U$CK+_rn3zMwe40K12SW&&o1x(JOtsUi)*Ng$HS05V{Bh;6t@l!zBohQ)0Hj&-ndxotQIXbpM7Ft&8 zuvBubTIC@HFct%Eg(#LSAqE;ZI$kxuS?flTZl^ki(@V+sX-b#~eJSkAvP9qr?I4G2 zAmA3DSpvD^@Ejn!{RTyRG4(;$X&Ap zwYm>wq?7Hj(SOjlM?%7o)ws<$y6+L)*A-3uW7iq2H&}1p`?0#slO!Gy(t*Ut&ft&^ zb`+k{FaJ5pO%)WSw4eEw)+@sxA#RcLM5f+}qDJUd5}c3nb16$2dO=0iPiv$&%a3wk z?mjAH&bik9G5S}v?jEhJW`m|{H0rf@b&Ueu3D9oQ?|7?zc1MOID$Z4}%Ky zUx{}=mi|GI(6!##W6lvO)f1&NZxA~us~X%9Q;VS9tz}lR>Z?25sQGiAw`LBpRnyu2r~wNi zm>i$`h~2PC*xVZ#mGYY&TAKYK(@h(eVK5P-qV~_GsvZMFtpXd(HvHdw-(`r>9cf!u zo*$pD899}b4@XQq2rOSo`!k^4b4%yo^0S(Fd0PP3L3~Q#`=$IP-jekp2GdK$0^+31 zA-I%DaB$$ZZ;7Q@060Z|qpC7YOc_3%>wwA~xt6XTlhM}qss!uuK)zk_V(vDczb);i zjVtEfERB`-$#?dIC_@&3VY@wNsa)0erPq-f^6jfi3*o+|;ki@~Nl4)Q z3(5YX6S`d&D(LBZNBGR6R2gEu-gz;jkP-GJFGYY57-u2V>M8DmM3OM*{dss$x_rHt zm5%E;YHN@9d-mg~!PBP?Wx;&o`8z|MTYlL4jEFXTUu}5v7Xy96_fOs7@V&=ku7w#` zu=X9Nhjfr81fSvlFtPV(W(85-PF)>pI7d}}@;)ZG=k2A53%97r z?36^|%c!f~a+66}9PRCKiKNKe&Y-X7-=!7ds#$XDy?%tmW|(zyXn3Kv$>WU^Yq09G z%;nD~-^stVCV)J2qxEje9ntUA);UWuE`rUi<`lASEb{7K{Ah|7HAETM>&e~yK^k_7 z`e6=Aw2|JHE7EgG#Adp@zy}oz6B_#nnA}POkG^bobg5^=`E0ynWDg@SE3KMEaap0J z!g3BA_-Z-w%tw;kTd{^#a)>C7iyoEWQQC3c`FZQOuxg)Od+^iS6$IH2gLiS}MxA{b zLurIGiU!8DaB${})#%|WnhHszsz;$sNG&+~6lQ|c**`~{_;N{Q<gHF*~u=A*0nx}_*!;_-!xCGY$3*Hha+T`%4KgoXhFejwEw#vVg z3`@kUaQo?j1+ujjtt4LQviS5+p^s%(c&rWq$TF)Gny`ZXbG}viAtD1V{35GACoeFkAefLtBM-O<2dn6{L|we=iQ7t zP_oL)gO$ia3eKeCa?=q{J7d|<80zNP_tRO4{duV?^Y|B(OZFi24MxwRNlgfWnF4ors^z;G>-uvx z!X}Afd|D;Rd7hx1cRHF zC%UrXbJ)AHR9Cu8=79YfISFXXs+!^A;H*`awpnxLQghR8to>G-VXO>ub0fCX8J^VM z9LUpv2Y550a)?aJo2k^Ase(vCJ<>UF6xavFw6ytp=u4+7bH1wO*TI$|sEt9QF~%<{ zZ{F!<=2{^8KEZkSv09_t?;!pYpV!$3MhsZgu@fXa9J0DTWPhJSv6Y?_PttD9yseCm z&~O`z)bY?!Jc(->Sc?9AFjT!GlNp&78xkhW9T%$R%M6QgW5NOm7kge$iC9>oBeV^L z&t-8v{>&;yPICa5l*`BW?w;~-sn&Nb_Gp(pmLQ(OX!6|nepVCX&VFXg-(i~GuRx_R zbmgrR>jtQT`@eCOMKvF+eH*r|)Dn=H z;869Im^5z#TUU3#?yRimu+3bn1Z2S=f4qBVYkC#mg3v|bCmHUFj!Mj0PcvaeBK_X^ z(2HU;Uq!C~v(0B#*~sEZXwA&);_K193)&HE%?@fT>mZ&o8hzL8WXTxAE_CpFp3@Mo z&!oCzwBH=_Br2ku5-R`4#NE29bT@pTue+%yv9a` zmP|}6;IW;-!FoJ%r|II3m67?~`_NEt%uT^Y513-|pM;p=Hjzc(?r z74a+Md&7xN?U0Vax~$9e0M|pB78lxf9Hl|i-S*tjpzXWKx@;AJNp0BxMneRj4 zM_ngWl9j&=GidYy{-3`w!d^S?yM;iVqH5j9J$mIr{0V3tEBF9R8$T`ulJ+TCTC{3UtZB*2s zw13#@`N#-!ky1NIpvTI)+Bnf>Qp3PfM5F&twq;Nx2@aNW1{wrDmH*SNq03+e1AhLM z-1Hj$?kE%xw2h0vKjYVS3S1(Xw9kpJVr*2fgU+6_@`!{?Y$isWvrs+FG;Y4LrJo z+u8@|Pt0AB)fa`=PhiW0#tZycByNbTiMtlaP=2Ce0U$%=zf_Q4Dfl9V`*+q#)-(eXJ zSG_2`yrD+}O^Eu}3WMKwY>44iU&35m^ zuij8eDBieOD&h?fQGVZ;T3Diw@d6iJ-7PF;qCOy~;^ zWQn~I-QZQsX;aI39Q{x;oZzRJY)x>LHj@L+I8~^<9fRd-*##~FaZ7eeJOwiU|b--f@=l_~5`LdgBkpzUDu)6|tEV^&VH6+7D zGG2ZAUV(fXwNh%HEGK1m&!c>qt%3>V*vt(23~^Dz6jhVeHARH!0|LD^bCtBJto`s) zoGKFGuZ+blE+ipMP8ZE3MC5&|3x-4tRJGf21DV-^u&^6BsXxB#vVtKWIeX6bS^l{3 zvNm|nsj4C7RPw;d_m|6q2I#Nn&2he64&B->g(G6W<$c^wx<`3_Qvl*`LCb-RPM*u$ zszR$*^_+7$GOaIrA&`7-O>S`LCS)*$l^l*Mam3M*gv})RcqCG3j>6w>3Ft(;LH>5* zcBw&bRYU8fzm4d*Ho;E*$$F`1v}}CAI2T8;JFzMAvh})pEk`^fb}FW4N5dSKIB<-y3a+poQK}0Pq z@$&GnqiujRMTd=IG>@P9)nKhP=EK~f^x4hxHGrU4)XBkPYJd0Q=HsxxamRgOw05LP1XZaqe>_xJeS2%k{%LpDB%)H&whBc_*w zyJYrSD3xDayzKrF!5CvXn(xewV4u&J(-LV;bRbBMMZt=8lCCM_nu{S~J-Xi6gKaP5B_KpV4fk-Z-I-(U4(ZJ6;2qV$5Wu(1(hAe2p7>G+lg)(gD-1QF|4z(d6|^Y=*i;;RJ7@ zH2&;IX;eeE6rs0?)YVF06HQi`UI-Qf3m~IuYA{K*PtN-N_R@2l8M*d%AR7+OBvXcf zs#?vC`pg2iwrt-n?9EOD&(Fo!BGGPvJbP1&sq$k`}Qf+Xt^1jQ-Tl&Z3W;{E@z3g!jr0&C7lR_(h6JjYuE$#&+o|7*EV~807YqeuHf!?zx+UE z#q0c@)%7GdJ7dVKR;^lNCJskY)kfElE=ZpZ@{V%CtaLuJ@)BAazxA2?Wy18hQUZ+Q zZSjduo2GUTAoSnSTm1mrrR}TbK{G{v3GPIv0V!IlUN*<=yIDfQZjQ3zc6+n)@sp!^ z(=|6f2!QI77TEBh>*L>{K0>9nM?4JK{D^paHh?ytM(-ZkUfn9v`+OQO!&V$-a+wAj z059&}rVV2Q2@6leZt!@L{wR8MOgr|1Gsx=zl zNou59pQ7PA51ln<(O`@rWGRhazRk^1fL}|n!8`mpFBXk@afhxcAlf9T`KT^eVs^gx z(+!1{ru(Rf66`f@F8b?j7eX&O}^ua%ONpl}&&N3AkMrBi(<+%2UQAT}? zn8!mUk<1@~=9&1gjl`Yiu-lwJ!PMh+;ECi(Z@J8e?6(FGI@P!yBXLvFPqYJjP4)I*w z0GE%=$V-*$io@p(%VN3KPVSaFlY4eMV68afK_`zMxU=!Rdzh489XQInYAAqR;_JP< z{Q%zj(88}-_|cS;chHQ2;fA-Zrk4GiY`qMTP$}F&5%Pl-RvH#h#_pQzJBrH@5$-La zKU^xMN5wztbO->v{=OU6%YxrV?X|#++=*hpekd&0=9H^P>s%S~*$&^@X};*Fsziw5 zcdh}MpRJiT85iL-zEHok6}zo1IL9oHjz###Nn{%=KQ(X&o=sAE9w#jP=wOdQk>@1d zVs=J^dLf~@75{~ zN+K^rb*yDNobFZ>9+I=n?`VPtD)A-iipGzupGp=53wPp(p0cvSzD?ZRB`eV_y)gN> zZekBHzDGgdEbXC0#uW)IWX4!#^P{?jL%K6DcYADOlF?@n&^i8lP5UcUoD z7yFx_)cTjqico0f%izLu284Z~9K^#-0fxEr3UI<0v+F?`jIkX#(|p^H4VqGc1Q%#w z#R&!!*g68elJBX&&46Xgt}Oa<`#w~0$9_ie7(;xEdeZEEDc$kTaU}HHJ$7QPU^c5? z&D&~VZ$0~!7XlrUUGcm-4*;A4NMpcmle(fTPwT8eUd4%{*%;=zMt~Tr-=$ue2T;Uu zu%MBjI)K*FfA5(CTVfKw;!Fr4Zm*jHVl}>TPi-ZJ)=U5mQzfUoGkA35d2mYdeRHG9 zLwSu!nU%%C6eJ2uk9m3{r=6Aq8*Q_JexQU(ReNK9*g!3_w;H_oi=XAedbPY?7v;Hl zZtmuw8kJF=VV~~hw?I_qS$l^>$}!WD5`5}qe{McY@K?i=*Eq=D<$VmcyPB5m%O~kF W@0(u>PGJ81N*Y7&8P@5$Joyh$5RDN4 literal 0 HcmV?d00001 diff --git a/website/images/logos/logo-redpoint.png b/website/images/logos/logo-redpoint.png new file mode 100644 index 0000000000000000000000000000000000000000..baf5a9a739240319a07e720a5601cc2b92dca5a9 GIT binary patch literal 6109 zcmbt&XH-*N*DZ)rqI3uey#|mLq=|q?myUu6gciDxLKj5oy(ztmpi%{C5|Drz+JkhE zW&%hRkX}N^8(;5z#yh@y|J^b6*k`V}=30BLefHTuPMnF64lRHkKt@JJt9w`TJ{j31 z3<(EPQ;=qs+s9=j!pFqGOp9cak&(#%_vi+KNaKHaNlZH!{FnSKDEvRXZ7jh5W`=?P zTltra1~Zp{SpOCD0>Nn@=6}t9K?@t(zxe+j{1$**@C*Ilc}V5(Axm7mB<7G$kfb5ai6ADD*#iV6ftaQ_xJdZ#Ah~vV1rB)y2<%)WC5a@> zL_WbC9)8mJ#m&3O$^Dl+;TMVrGkswJ4gf(IAaE4SL0V{&PvB4G0+93n6i>o`On#ve zAb1GGUJU|%1F@5sGk&2{e!(3+0a7XwCJ9I+3Fm@XNb?pi|6h}YNeM~b{r@OQ=1UOE zIGCgBKTVNbBxaXS;FwR41UCK?MJg#2%s9`*^VjT#)LqKpcm^mKI90g|T1A;nO**5qDKC=KxTeQU|0B<|~bL>9*-ZwHO z6CKZYY~Ilhy|znxvRVTZ+cQv#-xxHO`OQ;PNLYgWk?MoZ#cx z@@#~eBn$c3CTRicXzX3EaM%RcdzapF3yd7X)a0%qk5R+Px_!Nbv4z@0=}=~ zEj>)tKwUefOw1IF308;k`qR>Wt~c(ANt<%)2AuNG@)qpmyzhD~WPINURh zVK7eW$~1Zs=eMdor|Xeewa!jGY@b>c@Jgl8cSL`k_38PSZc>t-5HGVuw_y-WSFB*4>FOe)GbpJxW2nH^pey9zyGN*QEQ){mt95DkHA`-lxz(;^O+O z%Y}MyXdgFn-1U2pMMls)_tUFf?v-6RoV+~Xx;!;E_^co!6em}{>0#FuMXbc$IfTZ)+W7iV`5ru+)vpfBUP-DzQx zkoCEQS_~3h6R%^L+#_NPzDJ;Vgm}O63YBwiWCE+Kj{e96WpNf2&7^FycZQKYm)@$lxG|}Zyn=8?C%vw{bl}-eN$SE&a zRevcR6V1Cy~e zH^yuWclwUu;w6k8D}f_Q!*7TD}xBt5Sjjalr8gMo^Mcl4NNs8 zx+>zYW>|H-U?Ra(jV0!JTo*@dTz2*{syNO*-6C8`3O?;>%rXU?#V8&FuqU^ZQ+d|MJ7M)) zn29G90WPHI?m0jh8|*)YuZcf)wQZZ==%wlpJp zIo5#!-%-Ko(j>1t(o@jHRaLtzghC>%Sou~@tcGuj@{AmSA!kdKmq9SZgp>K`;5X&j zfO|rFruzHH)8!iG8M@3peiHkL^mfu z$|=;~QNEu@sW+UbN{dT2h{mFs&AG^p(+R|F?lF*vwa;hREu1`=L;hOHSM_{PnkV9c zcsdOh1$I9sF#kP2^p3|Ti(xy2KqnH_ivqxJ+Gt-n%lr(EzW|0Vl zNivh8>~CcL!w7T8!Z3#NMH)B2XHaSMu#b_I+)alYhyFAw69*2+x76hPYLOPZ{s6`s zoFG}oN9yu{zZgRhX+sK?Mx+1$tS?4jO^EQ27^FYjOF!3Y@6F<45x|9J?eQ5k?_=tn zfzU)8UE+5tJF1M@(=4n^g(rYt+2T~Bzo5`Ek^B_0%^*ph#Fz4DhL`2E& z4Mr)diUhli9r?SUL`g;mXOpxl*t!2}y~#C&iS_y^yn9L;*KsK213NrK3USfsnGmlW z6c(8K2lWAA_36;H3uM2xyGls5KbbXH*&LzlR7UmM% z4Z&g{f_nURF0_NEFuNh339Q# z;h~lZ*iHjsNQa(dS2C^3PSiC@0Ow3_OWI&U~%=F zXqo>>gR@`bV!Q4|>OL@&spA{N8Dx~|O0~VwvaN1L$xcJaZQkg+0WBR)CvuWhvnwEO z0a=OAm8JFD1smq=K)H#sCqb6&__D*R7hCF2t{(4{>bzc+{ZxvrA3rTG(Pag)d|hT+ z!qWbj6Kee?>wdp#e49#RxNh>&9N&oPm5dZyAf1YbVLN(PP83=)GlTk61Zq6#mt&K| zA`hOz<0}`vJ9UCin06%Ua_H*i2~Y2cc$*FETs503PRF3;kk~-rPa^CTO{2<-`;aot z{Oiri#;Xi0MGfWH6n^9DpT2z%A1W{okdsgs>^79+$Jxm?MCc;W8fEUs6LvFICl#?7H$K&mT#IQnSjwn^=4L}~l_^%@N^{I-+JW(aS%lVMIhAL>+O9|c@ z*k=t!N{9oDiZx~$FB*7JeJ5~aUpb}`oL0M1mB5x|v6XV>aH+K6)`NbS*T?4`k(l$% zwV0}4pMFauy0$4ek2}Hz8sl{j%7-4tL!ge_l^UKAi(!zaeUrXNK}swZPV7IBpCU;0 zk&gXx4aHua{HCk8F?g!0RqeDSwl#yJ=%eNHa_BqGw=cEEMFYh0FB1`=7^)21IFWRl zJEJ4Q8q;^8%*;#VXhu42ELbofk4M>e33|*VQ^=ImV`-twiK%fa_b2dgw=n^y>PmIU_086}M)M0uir`WV{;D0hZzhPcg^P#r6 zj6 zimzj0D|n{5R5zbB$f4OHz#H!x47@UJt`s2QHt3)W)|t&(Da3(((_u!d)_~_#1-My! z*0OB8DunA<=gilPx?Gbv+9*#zz>)IGu!hx4@)-wq+R^dPWog~ejZe`W>dY6rpI~Ai zCfN-5W|DA#SMz>k3GQTHaSGD`RW+>!Dji8}`{z}`D^+Ba8m-1)7t#10Ld#8X#o4P1 zc_=(m3!QLfpn|*+1^sU8I^ghxN_gc7zGY_yRj+_JaCK2y-)YpljJ!}Y^Lp_XL#Th7 z3TzePyOUkDGT(|V_*|s5Bdx9YtC_>d&C>iCf7 zb)HJ+wXf%w0$y9M%pUr`)__+^%P2XfVWL2D$!pQYswJ|oce)<*G5tE_`bZuY@!_gv z`0`P}99@zQ`#h4FU9i&qm&IhoV4u;{qD1-?j+4h?s(N)as$dW^W)sQm>KVV#DgX(T zCm?*C%p$O~uTqlO@m$wnOEOx<*Mkz25TYmk+ZDxvWFv2ch{MSbr`W((42s@Q$xL@M zlv~+8PBe*RpO@gy)%r%ScL(kv9Pb)0Ir}IL0}T%eUca_{Lvr*B#H!4BUdA3!)r6s1 z02*?8!JWj}SE=2bqK4hD-$c;euj%uc;3myrGE01E}-Q6Rqw-m6m5qLNgzO zeoZhRajDlv293+3h4%_snIUqC2Ne9X9ZeB2)P{x1CX3tcMK1b*`h@Vb>^g!S?@n{8 zQV^0qc7g0d=A)voV=?WN>>_V1!&Sqo?1t-=h92|_5AgPt_ryl1?%A=hXrBHc4-Zg| zi#_5M38ya>xek~&nGXZ)7~k+xuFIL8U9XmgKnpSQJ+k=V#~)?#j%j7;dTALY&@5}6 zxf!sst&<4*A3Yh~&1Ux`XRW&4-F)xg=u34G(!R{HsReb-RK2D5qw-krzD||#Lr=5E z)wV>YssYzhLhYWD)M zF%h0I@oDGx2oCxj+y^^9WnaFco7ZuGVG~?3Ku$V|ZH?3WW;(Cc`bA&VQ4^jG%&D&Z z>7Eqam#(Y(umf`(S3Pid#74SvIa7|1;w!8aO;edCVTG74s>>-CFTMz3(BjQqoWWsp-MFW}zmU zV8v6;JdZY8K`7{HgrG;=QM$-%`f9-g|2zxgySnD3{q1h*gaXsH(p9v+jz{KZa8rl% zl>EznQG!yaqT3^kAdl^-QU%jx^8-a~gx7CFRLpBPq(Iw)<&YC5<^*FQ2!`+}#-aG_ z=zH1bbAOJq<~>#UC|kEnFw-UlvQeZcruW4I6wb z-W-sXos!-AsGQ0h_#aD@@6oS>&O_v&AJ5={Z$7kQ$?kr7S0kFXvAWs9@Fg*b`W_uIP}A~w;1b_ZQ`~50GfyH zj1AhTqC)&g{r#$44YKTzpkqg7QxCM0KN|yiLh}_KDa;E~8l6Sx%F2y_gvS7~8=2mf z;lgT(BGaKOy$pn9**lesz;GHPeR_SlwrM3Q=%LHpg1YI0t(?}7-)&wV!=>+*f&5rP z_&SkU6GDCB;;3nd0A805s9GE+XON4(PoTJ&rJ3k=;_}4>MK8;Xd1uRtd}QQPw0+rr zb9o}l<+RRorB%xfb@lQDd7`h)b)Q=_*q6d5(cM6JYe;qtj{H?4iqEg><^jkjB2vhY zyijaZ*LSI#KD_#x+J0WPpMm{DvZL-b3N6*$o)pvPy+Rddj%LK&U4Vc-$JPO0&80KW zF8Qq1cwuH|c9RT}^B8oGQdoQBb8bQM8_BycGo6fz%W5%LmG*0M#OrOkU{Ad1ckGc8 zxTq5pxA8l)UBfBRKGCt$ZR8G(u;Vo#x=1(@mxQ8B6Cw50GaO@L)R%eun=@iZHD}!?E#yx%RAg!Cdo%`K(*j`OiEiaP`Ir3P4 zo*Ue4lJCSQVh1Xo8hc?6Jy{GaTAU2-7d)tt1x|*3@-sp`3hGIpf43bivsuqwWB+i8 zTARgn%;aX~x}6k-dKdTnZzE?J+IjS=8o8UR-iH1MYOSU^Qn#DVQQgYa5Q#&x9{Z$H z0K@p2yw2)oR->lMerLFS*f{~94r6F${hsnq)ljk~D#~BbTdYAK1y;o6`~en2a&AAm zDgjW}hj9%i>y1RkDVIy9_rDPoeV}|bd?*#NeQ<~L050ecT`m%eBLFt;E}FaP@MF}K rnEp>v{>##IB+XN!j$np{Hi~;Pm(~vGeoy`RO|PqEq*<+QAO61pGr=lB literal 0 HcmV?d00001 diff --git a/website/images/logos/logo-yandex.png b/website/images/logos/logo-yandex.png index f84f4c1b58d7ebf2b02b77d9a9251ae46706e7cf..92349036b3f14139112a6632aca28bfdf6ef9d46 100644 GIT binary patch literal 8381 zcmZ{JWl$VV*zK~oyDbtdxO?#6?(P!Y7T3TokYK?r!QFzpvn03&cQz2*gG=D@-dlC6 zz8_yr)%48tV}1IxJrk{|u7HgJ!TfE1kFp(XZOfnu$q zAPac;@5=8kO924h%qhvf*Y#gH$qC3#)4PZD?rlJ=mXK_ zY_>gXwhvns{$e{*U^w|FUq-~Thu>EYNpurj^PtAvDsAd7RrU6cXhMj!`f3(p6B=9z zL)+0>u#z$%wR&y^1>~(P-@J}$740BrvK7>09+96g0hFpdbLxa4!XgNtWn6tq>WYee zqn$Jhj_jnJIv|jhznRW;ruSNBZ2Q9Bngk3u$aK2}xU(EcZ?Flhm@JVK@DzMklv~B$V$s|JfGW3TLNF9+ z2|@bYEwD0L;<}}MZ98WE|I27~PkUx^FB3&8|H%H>`OIyp@~M~CH81}oHkxs+yc4rtR<6awH7J-x8 zU;da$;IM0$X?)o2@O}x&=5oGW_Cy1JRkmT=LF^QNb)}0;DWz9)jba%W1F64p=^Uaq z+X`G(<@6HKEIDV4one12%9D6SN9PA{a@@EDc-K60>C`Bmo{%6HMDeROqn@&?u*)jM z7Bq5q@CvoqTXr~L#>h(`Q4b$yFx$RQ_FrMU_pOZ<`mx}5ZL^NG-(~Sb+ZU=iD;s%?&L0lKUp!&qGG^H&0S5`kZ-@@x&)OJgz#YKv9DyhZjB37dn(n%N0X zTM-4hHzI2tb)`@bXkevwL(ISuECqUo19Y5o!`b0))=W~G} zkIT6G7O>48%qR3a`f~%OE4FAg=toE`zy^P>Nf-a53d!92=#)n&p$iuKwQ%cKgN@Tv zy|Q^4ir4$V!L70R=2meOpQ#&FE?=gF{BUopK>1l)+K=H)tB~Fuk_BJdkh8ZiA~-JW z3XOD)$kG%hXpZ7Lwtf2*4|WDsQa^v2v<}sQ(r14LrM5zjAX!jcQSwWB zqFkZnYe}aW*n3#?sZ=7)Z}KrHD&z>dDC6*A4u2t9?L_n@u=&$h3oSxyNkYB>KlZ;c z?|zVGo%Q59QnTqp!uJhxC0kJo>i6u{9&$nxvQ!)W;Ew&fC6|qPGe<{~(TP zKOwVLoW|odgQ7<%U1-5!i4hbqsn(5M5$ph=otscVSJ~1={p16_U{D*Pqo&)sW*adF ze0=t-s)4LpE!0vlVM}-P!J($1c=hZJ7sDcTO9_~j-$EI-6 zw{-!{OmGk+RP^M~3qR*3!6(ekf81sU`|JM3OMc?evwFX(Xt`>{s()Fu1?HUm-5L>? z`&^@}-ifMz1BHJ(78{JIN%hxXl~o_-W7sVuuSsZW;ADi5LxFX!oAIHb{HLO@bqUdqQ_E(nu}Zz;=d5qd!uub`g#CW>i0&fKlcUmS zY>6_qCAgNYPtoqS71=C^1<~Ao>y&+3L(D;`Gr>9&8+I*x%*Y2*Y$J`*qKjHiO4Tp} z-wY=O$*K+IU&^cyRw7_a#v3GwTK_jGt`yfwk7@PomoHWqeLjx=RZoSAb-aOc*Esvd zmpKVLpV~uW`E9|!RjfN+ZbF`VMrf}3@Vx9*m%Au7!{k<+FOhCs(~nHfBCefHehBO|c`$5oEWSlJTch%DVfplP zH5&HMtx=tq5Wg%V-xU4Hpx%NWx=p-#glk$pJ^6FEo4Wu__?PDw;|ETe!${}@Z=oCB zl6!@N`=GgVOe}d2S{uj%z^sRY&?{3ST*carHZ{h$6RwaW~@iRcb>?{6<-?DM+s zR2NlF8(ztZ>W3p+aajMI`x4L%yr`UqYljUDv;cBOV_|R1W(>aBrMWpNBQK*W?L1Cl{>hnXH7ItNX!#I z5&iRT@Z257Td9)HZhv@t*L|yMnfICDPIH?4U$!OAXuBe$on#w5x6X>Lk3(icz3A99 zznt=2sgR^~bKxcSW+S!?X3xvWwd$L=_-F?+y|nJ-vw3*Ls$&S*U%Q|yD(^|en;RJu zI47M)&_Crns+(|FfcGYq{!J@bg zOi9Ui#13&T72vuE>VFZA{$nx86kQ+VdZH?dP?d}Gj#w$k;Lud&yz^<3F5bv^4pEY8 zhM>UeAPD0Eclfb@wOH{*MCHbssOtUXIwKfa+l$;E6%NRhpK5#^3Mq9E=d^Mb{_p)~M+Jeq)l) z>AyXi%Se1hmdnltBXibn`E#yL)oQ)G05ljEwK0k2&o{-M;q-3*1YUvgQN=Dyst&RadPZ zI3G)(h;@`Tf0>UNF7pqE&Z1sucFbCQDTX|EZuUAR6dMzdPtn)sa;3(@hG_j5KK%tI zbKg@9NL=II-P$uJ{5sh0=T?(bO+O+K#%-Z#TePAOLLfzU*(NDVasmVT5Q{uOYjH6) z8|!g;3Z^Cb*HbLM?DNRkuZIu3k==s6+@nD(w?#G%o%w5TZ&eRA9gg)7!alm}gw)by zVt>th{$|=s$QzcHR*I3rft=)9djF_VV1IG39Nz6!UzCz^Y}VhU2#nunqc$Z27yDl` zEj(6gQ1<9-UBYItyODm8(%#M(2dCX&Rto|w4EzVb4NzA~w>wOgcpBuVG7cuLl#|#~ zeD>v&XOhfpUURaW!kxWp82qN;E5GJjR9XCw!m?i5M24X0TX$S4<;)j@E=(RXCp-q9 z`PS)9!eGL6kd^Bbfrq9NhGS~7SeuN_cWsk5EKaJ%^A!38q}ua?3=$OK*}$~vdWH3Z zN@i(%uvr;b0J(FSW5r&z4Qgm>%*C(o!@_e??;NtxC!auQ8+IHVvf z^_YL;JIu|*d3{#LL;0WGYMA=A2>h7LJQp|bg+WxM-Ccm?y zM2zsX!>>2(x6cd;^gV$0hOz!hBx(-mimt|7=w73x62x&_&~@K1?aNSZK2i=eD>UOo zcA$m);_o8OTx};N^t9|v8P(+4Pjk=Tl~0JiL+1+(o;-231jtE3<;6%+V2f!h?R*hE z-I)BWRNSpv$BX3qB3M)t4-BZ{_)IzZrifkm#n9=70RuJnKsuq)Q(LwH7DQ#0_&V%x zF5WnmOoS!B^U?~RT{q(2(ccJu9QA(ox9rY@47?O-i;5GTa}xgIKy08rV{6u_GB^_+ zklCBqw@cY1Yc1ODN=8Cx#X$ds9?pcXkd*&LYHHgLeXH+d_*Ql=@&%W00z9{0MXTJI zH1qYC%{OQsCveNmzbIfcYfqEW^4~`LZ9Dn+OB4f8Ek?9CoynF!xiG?Y>mT#(Ek)6s zI!a8GPH*{i_$N7_K>y3~$lO zVG)X%yPHt6Vg_!F25gzf#9&Suqk8G@zdVlA!yt{RLdLi3eE42&OYAZ)oK2eYtno&M z<2sq0D+9J(iWnRdbqO{^H!=22I1U9E!n2BlmZ_>_Ee4Zz@k2UaH17;|x(FoOd7sr+ z3k)>>UZ6dS3+#GO6wFQbN*`~77|!f9eX$DMYazyk!PNp1?~k}CIvi>B)&tp;?RaCq zS^BMv*tgW0jvB!n?KlG%e}qUrl*>^~a7A6)C&vOeXIk+ev)%s+i?s?W5qB{{ z`Zy0Ng3?}}p%;3n9;&A3s+o{QL-?lHLn(B-Sa|AWe7#Q)`~`h8PmV$!=#?~AL$q!G z#_zDhn<&wqDLi^P>bT_*GKfwp=a4BC2VeYb9?~zTF^vpsP37hvy;=EqvQ$DXY#Rl# zv7btS5&Chfcb{p9Q9pOQ+&o5bAH}BHS0Ihf%YG}ptUA6Y7&03L{_ucADZ{!6-&-^{ z9daYDttv7vQ1JwhHKNf*`+WdTs#=I}@s+w^fXk(wADaZRHmaSslQTvgQoaLkWK9SB z6o8DaMOUn>LD)556y}h<5+3w6;px$uS=13>BF1LrlPiq&JU`l<01(|%d;)&*p1WRZ zBTq(SMw#}mW?YMQ@uci%dj>CJ(poqw-92&8?+RIjvIAkfNQepL_QNkY_*MtQ8?*gm zEK$J6Zk~;H zDVT~-AsIT)6SoS(Fh$ft%L!mE9vMw>MmIlYB?sa__>GmmcG_vTr>Hns-^iI-k^=n@ zAGanF%)@6l%NP5nUj%Hq-OOPhGuhBKtqwTgdsY%Oo(LJa8~g*xrI#==;SrqLYNfk~ zDrsv~bH6uUm}374pT@X0zv?E8S*~?K9{H;kLABh~jFQr|C^o!E*$B>Ga*F3nUtD}T z=Rc&tCKYLpRTDrSiZU6JzkLxGCyCd{Sn=(E2YkmY* zwPLDLQWUQN^-6T1kY31BMM9ie!aXj7(?LJjRD1)8RvcW&Ikh>=++{rzklopAY*+sh*tmQ4+UtjJ1&=o4UhW@C3Hz72}HSlQ2dvld$&L{ysh8 zLe#oYG7R|zl7u7FOiv#;{C~i?EnlK?uc_eF@LF3YIH2KOje41Iaz1JN0yS1P$*i+! zSV&GO$?fkm#9UxQ$a-2eWOl858&A*!y-grQseJaQqn;Ko`4aM-V(0Irv2xuhrw}=X zOI{h5pYRnjb1OV>rF9U=$gVh%Fv8d0pH|BTODTVi+gT|2f!!b97EH=14kR-#6I37@ zgn+V{Eh;QPDVS9R_FlV~QeOS*lF@L#DAy{KcqHoUqpCuF;fb&;4+f$rHqiuq>w{J3 zn97HmU^45~pJ+T69!#o$Q&e+yQG`X?-9pEs!MQ2EC8_Himl!SpuENrF>TCo5Fkn-P z^(e3l!j@iby&q~8joEez%L)nb=-#sGCNuJ;sB2xtoEO}ewIr%M1Ua`Pw5jtL@n;@U+y>NWkh)UA(b`=n6_QND(So|AcYwSmzdthkAb~HTar8%IVyq^3irXCK z^Mge8b#6EKVsj%RQUDZy`4V@O`bsaem&2YFgR@D&c^~Ns(;(mFxinToFZ{99&!!&+ zUY8TzL7p+z_*HY_wp=VQVqYVrEhLZS)|CK*y6=?;3q}v}SLHo1+k4vquS3{A8Y8ET ziZIu}X^-tyAszrCTKV?n>$@`M8&?3hnG{SxtyJlv3sk?{C?M95NNtS%E60GN9Ue}9 zc-aya&8FUTn0BHO@W*)~V*>>2r<&yQeF^dS2bdID~@+3At1**C&`{@YLExw zESHBZ3N+V*nEUE=K@gS3N(Ve>Z<}L2q<#*3BYt^GPTUahE63!1CJEVeId^ZRY4>Vg zLbh_>xWw6hjL4$@`+|4-z;GU=E&7=6#Y9X3+HuAL z)AuzR?j%;OwZ}a@UhL}+-xUy!n!}PBw9wmdiB>^JuW4^M!~hzg?1~MT#Hh#yU4>Vg>BDdEvKq`*r30{RU*ZKdv_?FGUb08Ktf@BLnRu%pY<861l$Q z=+QHqS*R+rFm;0Xg8U5pqUG{<2^+yU>533e{`1pI@e`0EkoGpx946N0H0_Pyz<2x_BS3OYeC&elq7|kXWKXh zq0*x-rm-O_b%I05`3A%!7d`B=5xtR|%OX$0~%toLl91 z@}HyI1cXgqU3kDHM_o^&8JK!vo(PZhL#P(7*Un zys%p$Tgyv)JqJ!CydN45;PQN9i1vn7u0WtYUd$%)`#F*o{+3elx1pNBZ-7akvi4qD zCEHaS!hXz4nm-}M22p26YN@}v_wG-1Yn6oHl*LY$E`Q?fakMM#OR_|}p(pFv&&tD> zELa?;L=1N(Cfh5-JgxsQJaYXf|M}4-PuP4Cg;aYvw`G`QT`Nxi(}(RrsHQz0Ml;m# z%kuN9kS5vwy1$^Th5Lh@>X5hkXDU98Vz$B8G=!s#TkX341x6E=p3yb`tV}ZfYC`DB zo-Cv#KV+j05Qv!e1>Ge-vIv3jhbjrBoi>H7TAic63~?p0bG&a)wM4cHh4{zb8U)sE z=g7bNUS#o?aTLB5?e^Rp$Ft-rznz`#%j}6j<&$6iYH`fyPOM}A%vraDix|X;{_qaT zk0bz9Sm!?9!`VJ4nG`Hi`S=GTz`(umOL_(*FI`H~1geLAI28$_-M6D)70`~di5{d5 zMX?UV9{m^PS5ERX(NKd6F1_0TISi_prr+IL#mF`>pu4qI+Ev(}x1ANEq|}2(77E5A z_T*1b%0%(ykT9{m`8=s`D$otVOw!YpFI4sbaYw$9Y9vA1)j~2fxPH<*v zBA{BAKIV!t>tbZERnL#1?Qvep!;J=R|*L28jh3@kw zVbZs9!`v6wBYU?EsPMpnQY1oZWd`10aVgD6Jba$8Y zvoU4M;|-~CqYMO&w^tK+J~kohOuc?|A{EcnUXoT`bn31GibU7o=KG{N_&So)2_}wv@SmOCR0SaNfXy{3m z(>=i3)1xJyqIO}qRapHV9*CG6H-y=S)<(F5ekagjOD{MmI$6aXHbpWKtnEk}gw6S` ziRQXqo*;y*FPMR9&9TY7g|(Uc8?xd|gm_XVf-~Vvt$&P!H64CYBa^K z$)5}A?z%3uBqJMJ$fmOV^dstio@JP>acCI=kAOgD3NVfM};BE;{pcA2+Irc&prxU;M2?QI&RsRiTmT_)$X6nj}Sp=9i>F6*?6_CV5Yl@eaQ@vD+cg06fgJiaMmA)ZJr!@u(h_MRs4Y>v<7D(l}XxoM*T(~nhDu4vk&sgLrHf=5{z3tj^XTRh)Ivp66z}V043*U zJpX0aj;Rz%FEj1^I3ld|8LGKMyo@XJQ`k!Qb@Z@7yG$ReP|JjRr`eX^`b(We(+@)4 z7u?!1AxpcfIEB|g{%^hI481>CdE1KEc-p=e0A6lh9u97P4sKpuZe9_7J`o;XHg0Ya lZf@d7`N{vG;OhRt(JtVBSNKr+?D?tyP?A%Zt&z42`#*6C{^9@t literal 4152 zcma)9X*d+#zaNB*Mq?*S_KX;0nK8tWb?o~#$QpjgAWM-jGK{T4wzBWCmqA%)ERm%w z31RGnLUy@&{?EPl&Hdkd&x`N#`F@sjzR!7ap6A3G8EAok*MI;30HmX>ZUO*M5H7Ii zWvYvMl-j^}K|qZ3%rq|A|9zd`{I~k43eUgQ{wLu~`TzEx{pX&i{}1zj9da?rzlJ$S zaGopk{i{7y;XX(H~FnLc`hP1o!+jd@S!-aLoG*o$$^ud5?`tW`xA zb%r`zfcxvg004uXj=HK@@VCWpx_*&JmgxBxj^!m#etK>5K)&cwD_m*son69-gZgq< zDNCc4!ePQ1UAXa{cl$?eP)?XfEk&JuxRc~zQlRFwLo>E1nqCt+_IGSIwSuQ*@L|Z8 zD0nDW+1xM94Lm5I4?D%eW}Bn5akb$OjpMEP)mY5;qsQefQm=fOw@U?jyxvQEP|GlY zn`wTRw)9@S?5Z}u{|}yr(#_Ljlln8ZGf%os_j>E~&@1?6lm__9L4QjVOb&F3B2<@)B;sv<=gYSS{WnF1aU|eggi#Epn|2)+KX^F3 zeJ(s|Y}TChL$kasTp@Ab3>wtZPqVdQE1sjw^qe*lHlqN{f_3>S^@Y?heB1vy75~XM zw1OS!=}AIO%zQMf5As{SMm&o0t+$;ymUnElZlF4>R(f2O_;ct>K*u=4%mU4yKCk-~ z9$CriTl+xyQ;~ERm;hCH?U|I!J~F{Sc(`LWemUrL1LxoQrngh2Z`=I-nQ4e0e#tJc z0dfU`~o4(2F>9pV9`yX#tfmp z9^A{6$lW?xx&G~8e(zx<7i0NX-vIJ3Rq2p)#-^Zl=aw^Ny$%Ay5TZF|ajWTUnmR4{ zN+qB_x}3CAk9Ou{H_~-3XP$^s#WGEq;_hYlsH3zzh#;=gcB{{urcO_5hc;jK>Calc z%v8aLr8m&aPjs>b6pLY^oOoRYu9D%h>YIv@a730Kn}vev$0&ekmW!3NM%fXoPHXh8 zY;T$rYkj6B`nwv)4mbale6ah@V*2?&F>HlBx$aWroxpYLayDH*Mj+z=XR&LwMC_OP zO+bI5C+v3&0s}KjG8Y<(Okte)!0DH^viHiiVLcogmS)~QYNn2P$kzUBBR4L_EW3u@ z1##PIo&|DJ(>V$@V1_t_@(a)2pKm5>4jvO(pX7bOd%9pgFcjx&FsS-ON~4Q3I^l*6 zp2dB?Zy)cLJcnm68)v_U7_kvFG0W1$LPB{U0ibOIM!0Hj5$HN^AA-n1;Q$J?QSLz zzRUZmpNwftcrP)^NjoQI5Hp1~8UFEo!`2UT%GV5{?ED&)ZTy&U zP~+tx*~EW6yg91>_K)fKOBgkmXVvyJp~|*;36VtZgyM+@e)K+&w&fKr5eHxqtH7YU zge7_UIMQ@NdW;-c&wSpaN1}?VM?J^e?UUZ`pf{uQSnkPC4P~p8AdIs`hnG)%I)$si zZAIR-?{iH)UBWDBfJ6g1AqubH3!|+jx08}T3!aY^TM0=(mmiQB77N2;MN0;m>FK6k zRwT<{W$h&<;g zCGGar$ht74qW3;oV9nRxu<->32Js~d@oe1Hvb+lJDTg9QdhL7n*&9*7P zTYNo9#ua`hcdo14^JqVXJt#X$9c@$1LXz{EgQqc#(U$7F_S6x0wgL*^!aI9N z1w37KmeVtnS*MVW8?)6B%$6TBkzjdSp;)fTb~3QUWGJ65icTraIJ@6|FmvFU^@+`E zZ|9$Gd%Po#kDWGbu5U4Y_b2B=v=`l>;<2`Xp;$hjzHy6Sv3$a1IqP^KXHHT8zZ)=TZVq;Ov*Na#AA1_Xnjt3 z%9mK2%7&afcPQVX-61>(lWRkZdO4u=Z$J~VEQhz)i4XcFpp-+;@nyz9yOggHPgrW- zpF70;!x#Q0#KsLnJyBC2aYkW0XH|e~rl#~G?u4_jOK0;M-73+HhD!NvdG`3M_GSc-M2a|SL#rFjw{xb0!<5&v z1v;t-ariuodd32R1dXz=f8?M_UW~|iv*(|)DT%8leYDnozgfskVi<{qoh&j3DRp%- zYgaX$?Ta45P~G%XTX@v2MFRKIxhuw5p-GTiPq7&poy}F45r7&n4XPb&Gs)$cC1@Mr zH_C`_?kI;s6Uc6*NCOR@kt7#acz2z@e=Vfo zT*}aZwh~{#&tK%w^Pm#95i?yrsv+(W+oXtUk}dlYR{6y;R@YZrj7qM6PzAz7GcaK< zm5Mp(JWnz4*uFLMt>LToU5><0=H&cNw`y)^V7!fGjU-put9F3d^p!82K^@+2 zc+Jrf^dVeQL3Y}}z7T5t6u3e%K{9XLcWs}M^y{Q^<#<5H6gAl2E3DYyz9mI|?aD`t zT$JWUP21c^RE|Z=4S^HLN(jfQc6(Wt8Tp2uvZvbqsrE!?yuW*U1E^K@6IsmF@wz5* zl7Xxy3?)9yU5y*ddx%kphBy5*wttFP=9gkX9TIeY#7kr<;s}k7NNa4F+0HQ zB(snn@Quwm&5oEiLeTAXLq!PetADocJ5Y%^=5_YK2tG@UbeoQMiZeh6BgL$?N~xAG zR!Ip+ElQUC9CyOucFRt&nGiE@*Wst)H84qdEU8u?BLl4JGt+jk{k51N&t!kyjvF=~ z%p`!E)0V546zFv!%QRVg^bI#UTa4ge^uJkEb#5X#2qIj2FOy9WnA7gYo~sIVa6(b$ z?owJ|O>J1r>!7UTz)`QbN7sH@3Dm2pjikY^LJa9or_pQ^S5yGz<12a^<#)z7rZkFT z*hs0#;4DC%i5kil+-MZH@PeWd*YsFCnSe^Oy~o=A@>J*LO<`VUSV7#;a*`Fx#6sHH zgF1s*zB~3{OG|uKeTgvdsm`wfjVO3oQea~aMdH4Okfvw*Q3jns(fm6P!)Y+*$Jmb4 zHsK7-#54)asea^==0@W?_<`Vi0`R<`z-HqZt{heuf5f_dfR#vpoBtfiZycYa(7A85V8M?TwUG)8C zVaktKMb+zw{@$pG6`sAG`Wp;`#z|fAi`UARGx~IrHP=l4A%F82z4cDihg-3b5O~i4 z^S~1NyY=quLswmHS)+b6@PQN-9(HnUWy40hV447TzEO&>hMeG{9BHG#ed?v~rejF$X3T5ZwS%4 zwq`Vjg@v~7h^q74)YH5#=vw)4_$>$nTTqcM*HWT1&s^-F)D<79)-HK;cSw43)5JJY z6%(!b+23dX=xaIrh_#F`@WP4e1zc4ABS*UHO!#6P_Me{N2|V*!#%n>Dq?uoMD|T89 z+A~aCD500aA^)lrN4=b)%riyT60}gjw|!#A&sw}vkO567pBDtjB}Qa4ky%_b=ncHd zs#T_vw%xq55M67&X8pBJt|uQ37}60G&hk3pnRy_AD{%SR_`;Hzq6(l;%J&QIL+j|9c)}pB0+qzs9#H}~1F}Hqsy;?+5Y~bbuV8{!+XWEH_}Ezk0166B07Srlx3_Zumb9mpgD(II0Q){H3;?|C z!i3v;c(@3%vpc)9S%4sxU^Xj=6T6Rv3p*zp2RlGm%*VyT$^q;_VF|Xea~7ex>g=JS zumg!u>G7y=sJKXhZSCa!+`w9Xs@hh54pssnDlt(CVIM&sCl@EMhXsX?lcTe{ppOXE zznBZY_x~AYr=s{5iid*;mBc?%DfCs;DWo87U%|1Wy>ccs|>37P*9LjN9m zkIR3$|CJH%H~*DWVCVNla(mC1w+n#$f4hO3jRU|hsh|S;PnH4z7$i`J02pFgITC09 z0Q&v+FBt8=V3^OlcmW>ysbE#45imlAFpo~W<{7Nm$v3s)y@sy*Q$kL zT3Kfpjt7$}ZAlWB6ALFI03!lr#vbPI9!Go6Jo!+308(4R6*#J;{A8>)btmN*TTC1W zyLA<+9NXTRYS5(Z@wf7X59LNEYjtP_0<+3=+U|M1UhxIwaxJyG7mZh)ccUvvjG+YB z1e^~I0aw%*c+GXt9WM~Q^9h$Yh}EZX>@kF;ABF3Bg*P|9veOLUHxn5`92K&vh7U{( zNKPeMVWid`URxR&zbtS5Tv9do0^xzI8;?k=cXP?FXf+lEE`|jpSDFCxW(MnsiS$up zl3eOHZt!F@oq`T8ZbM!r0UCEX`f_8SJC-{ zbpi%wvXJ8j=ZRB{_ZahWD|&6&M6GvDxoo`S9EQ;te>`m(7J|T3}-%Y(I{e0I+8NLj)JO_$?SkRjGGnm&TGPd^5m^dRp2#Da;l# zg^2^PGWBzOSO900FM*)d2yikU+L}z;9=l(`vdv7*C}}GTP?We{@T6Gp4e-OlbWnnA z|ARlna-_9L7gO}IrAS10K-Y9G>_>2XbZKm7aEYOwqn?*jSlaDxQb*63V`q`3ugfsH zb3_x>)fT@4Q=wytI8I2I>8Y~$j^6-F1|kHg>(AP@T0oQ+X7lk4pm0$Vo+KwP zQTM>4w4gw=EV+|xAkuO<-`A#em<>n0ls-P$exB0|ErG9ba!%Lw_;dlruNhkPU%#v8 zDnug*+Wy}DT!`uwUMI1*#ne1Tx5H@(uw^R8@e(C&*3ajP(&g+lOtAt<4HBVcVEY!K z@UTay;ldj0CQDRby+HLWKHGU#O@=vNKb7Q*zy zwV0I_o&U}+TEbep@pU0R60dfL|FTZ`E8QQQ)PyXVXibZsUt zGi&OfK7LUgq$UnQntRA83@n;l^GN8xwFsh?)~GCa5>W*S$56FVy@NI6fXswoFb0DGwdP**)4L z(7A(dwX!pqErQT^LllF&5JxG>HLQt3i$VY*VB8DANx+K&tzcx~(PYDfIZR@@0_sXlDoQLE^oA5WWm zSYzZu$BnVF*_S|_93J#%0OBed7{#U3Vbfc7F(xG%9H;Y<@l@pv8+gt&ST-pT!a3UM zYrxjE>V9K9GY8ry*6L&A6|7b?n7n2^*}!;iWN|Z>KJtU$W#KvwEj;j_3nOa@UOu@! zmG7IaZT}&QFE9VN4fO`lv|O{?uoxOf=F*TKy$wt83koR)q~;mKxN#PQBx)jFRekva zfb_+4qal4?B}=>9N$p%RG7M?T1=lRTRIp@EEqCt+5HkyTaf?q%fZpA7Yk zQLBcjS*-{(+}0Y+jMmz>lJgxY99Oc>1vBCU!@3vZjU?{mz9iB|W(|7O?Fp5WYh2N`!EWpfxjcZK=4UL1&$Wz645t05S1-IZ+y z#6=f<5cNPSRZiqoDC~YRM!7jcf$2Y}9a^GM}9VQ{c~A}=cPKj?1l(bAy$+dJV_ z_L7$pR1(P$8Qw49adTYkPp9_&tY+Q;kCjMMHvOKX8yEix&yn8$!0d)AlYydC3Z)p1 z<-B5{5glM388vMvs-eorwY5-X#V&#Bt6~iTCF*n8|0)Qw#SHeJO_r!jY@Z2&9kFEB zvXGu|_KT9m86o4PjF@>>25gNhw&J}J+rUfchRa?RT7<{D>*BkN(0p25dn>2g{VI&+ zRXjA|+IHBe!rk~av020{Yg2QzV*EQ#KSRrr;?GKNN8_Inl1f1FN(Bd;YA-)QR_ULO zxhrecea541b--j|DkG_4zs%-8!X(~JEU6nC!0-W`RdkZyfm zV%?TY1Kx?IqrMB8)D%MRV5hYvIX31S;HDY^gutWMO`(pIVGWY|7E;*ua#7pmY2{a$ zw;kH92|Qccp;3~70!f+g+);iZt6?q1_~z)p$lj1i zJ!!55>jr%j8J?@oErJ<}YVGV2%Bx_{ z`L(yHP1judRH z116fJn*j@o2(a?60qgi)#cxb*d-dDN(y)ta+rFZ8dN|ljmwhE|ZVSbVx**xmY{fHb z?Z{21Uz*~3*{gzk?&H}M3L`1MWw@aP0}4`O{8WLMD(v=joC^?HG`dL!iX2E<1JdDw zu4NVhQ-^xcXG;awz4CKe2@7Xnx%|-nN$Y&|$l@P&iFm5LzL?+!E)eW`Swckxf3p3; zd|=CUsqoJTxW{F~+xmd**AVAU(+N7++1$U+5erRiVS*@Le`J4}?rU$qI$wC+x`H>9 z-vFmdX5BT95jXr^S1sSyg%gkRqihwhrF%oFcuubR7FeyC>(B`PxC+E8m*TMWP==XLb9ErtV`eif#bvbxeLCIpUh%BBu| z(GWC7Pr{&-RDt_qcdu*e_=)n2Ui_okpElE^b!0v5v?{Zb$y4I|{BQc{TLWcKwQ8SQ z@*cDC<^+eH{O99puoS?Z&tK&+TA}G~C5tdEr>0pL)`J!U`yRR!w zw1@SWGqP~emLnrYGd~ss*^0>6Gy*BMTAnk!RRtHDHWA`54`j*efiNMmWFn|LDS&Xv zaZgYqE-FFxT*@sIrta_`)tBoytW+~SJK&yVX)Nbtp9y_D{+bqr~k{)`x8+0_4EO&e>97V z!;$T!Xe$PV@~`eH?|wq!J)G1~?EUYBD_cfICBa{XuU|LV4D*U5Yb)e`bYss$2{a z{Ti&R2A~LCAhd^!7*=~>e=l+;eFMx}a?OF|8WTFWTWzTFt3y70@+=#BInwpSC0J5(4 zG47s{dwDsXF5r=cino=?v%aE*=EFK2GR-L&hm$zlNd#=I-EywMnNv7s!X%yRD8>2=m`o3r#8)OO*pEc%!}0x5oT!9X z+`r+5G^ddsRe)SWg)mm~nN}a7V^L5ei-g%cW-_I*juvy9KhDT8E>G6e$5!wa7m-O; zkGRUu>6=?xw%ZZk)r;c2(sjN#38X3h`mEk)W1XY`b;HS2{iC~brQ)-1Yg4^LE{4je zM)}23nL@bJii(hICApS-pE{I#?M=Q7R6&~e8=!Ps<>tD+VDyLhZ4gG9%tfFl&*9bf zH?rY)Q+(Ch)?E(<`57v9XraK=5!S}?enJ)k4fk4NFJxQT9FBU=lUcZ@&LQSX-Y3kt1$gXBBS$w%5nk}cvLe%rb)5St zL=|yPPW%I#Jju{-OHH>wx=F_?x>mPSAgUZp`LE^tnvsl#Vpjy|Fk@T(9M3uTUhVYb zGfxPqxdfDE2boC|Jegx>@&Ju8{*O4ky{|Xrx--!ql}U#ax3T4&vAfGT`aj;~{l;_C36w?NYi=V-V(Q-SW@U4D#Hsevki>RUL`0{7>UVQD&g zU@@66*&TcgvTidk%r-4bSG@3rgDLNE#S?2rgO7>Hsul zq`t4cyw!8h5gLMm#uG-#Jv26*o{oD1Xqh=a5jzc{5Um(*{ruk5n!K>%jVe{-wy&BH zn~Z3jEywDlt?eiWib-$F@^(E`*}xOdXTz?}ODf?KCj!Csh$anq2x)s4 z&ac;}W4_-gff@YTgnrDuOp)okfD(Z4?0@~Ar%=vZe13!K5z?BEE^1WQ zx=6#4ep{Hm$XvHtP-m-LJCC=PTedJGe@l_gGE7}VqE1y;9eq50>O1ct0{4(%n=&e}p>#U(p!6&cc6ri+W1+Ji9ZJmRPF%3OS}I!2l5e^{ z(CkIF^Eghap*^8kxpj~5Ojl0mVA*aYncFRzjKg3vB=`cj)Q?|b@bIOOmymZ7)tG;I z02+qcD7tQ8IlATc2SkUG;G54qLStMiZdBdztHUA|FLu6aR}a!<OY2cp{sd_X^P{ zOQl~l6m7px24qa7b|TipzKM(2fnm?+5oP~S%y1L)?40N9%q^bWel#OU*E!kK9OJeE zn%XgkcxF~Y5=z|AjxDaB)^G;kLt~EBjheiuEd+_aT%KDC*7 zO-fP@Ou4vCQe{L#au(HSL-`me7`0bF6VWZbTR#;h*N}~4S;02&kpNGYN}8CYA;z-E zz}eOp9%ju`=VaeqQU#uilgiewufF;BBGYF_g=M}>ZETs}OU=+#3T=oz))gBUer7ld zLzX5?tl2Qe+R#GFJUOaSRS2%(?c20^(AE870Fj6J@`La}{&lz;_-_^72R!(PBTs|# z8aq=k_rK9J-E6TBZKq691gg-&Av{_(&>9B~MdB4(O90b0d?y60e(q!@86Dv#BNz9ot2=Z?ZSgXX*xa}<{(jgY_`>w)Jt;94Z82f?)70*`zK8%B z?OcXOy^4!U(0)-mgX@0%XuhT5vo&c|hpRBVG^Q^SxvU;hlyrI{+bh_k+dC(9_;MYGmV{Fwg%`)z^N&MqBUbMyUYLv($e#n zmNa;&H-KUX+Zk@4bdAg`p{tg2<+E;nD?`h7wSpI+OuBVW?yX@1?AupeY_hc9f-lW| z9-dTY>ce^@0ZfbsmH|Fbfuc$u4J$1vKaSBf>}8>#WcFcT*2g3{P!oBmP~A|%Y2bjA zVxN}b_p`CYy<6n>VY*C1k@=^A$BxW7i-M^|?kG@XDLsZ3{z!YY!MH7R;U!BbIjs@D zh<+}I8*ZSsq`_cxX>KQIBhZj@yreK(17V;%c?_VL*DKoG?eBrYBw1MS0A_}^eYMtL zyD^(PxH-IM=0)x}-B+EFij3rf$77AQ>FI2yUYy;Sj~bK78_XRMKRfYMm=FN5SU}U> zjx!TaIo2GR^csWgu^Uc7@2vS1MNDh4moAN8S@8npo+aAVE(rr% zqMG16ejw}P4CG)8@T0?WF#ph{JRhNDp-?>3{o(tMIHRI){B}Ta@$Zeq;jGifN%P#5 zuV8YYr{>sO3q`E;1R^MH)9TNyjoKKor<=TOq80IJgD^-c z)^LsdMz6YHE}lNnBK1#4JgE}OhC7cXmu;-;oZROqT`r4GS*tx88Fv13!N>P%c~hZY zkx>ryNo)3wWnG%TH?Ez^Ih#!|7`5dQ#?*SCA^-0_es4^urTF}6v4n6+P~Q>SXPDla zrahwbwlgxkk#W1sUYbmBgNg5LUV=p8MtN+!Jdd~^e5J5w||U0oF7BpVk`vrS*+7`%L!%Ez_4ujk&wFpsHZGVxBTR}>*E#~p_E z9V-_A3Va?fD(p*n)bLJ^F3gn(j{Rc`P9g>og58gRXZseAeK>@FXb|)UsATOL-Un;s z{c^3A=Egi27wrG#S$A%9EH>#-kt%RGl;JKW-C%Y^iTewkXI5LX_r}MZJR@_$`LN&* zz6Gl53vI{GKknn#JG%fun{k(~Fp=7(rDDj%%pu+_sv{?8`>ZV<>yN13Q~Cq4+)rYV z*aiOkG0sk`mrCQ-S%c?{Wi`t!v8a*J89$>1&0HZi9XLcY7Eh0}@uTHt*s^*!%f~lB zBAs>L*VhdSB!o9j+%hfS9W%!k}EUiltZQG8u%Wb|?kK`;m=no$Y6up1)(pz&G-ZUuh=i_O}vh!&q4 z2Tb`DYIhh`S>c7NVSODDbBiY=$U9}p*H4JF*esZ-K{G38Zd z2MRiVYX2~l(#Y_lXQs%xF2v`mZ}q0m+ei^b^=X=p)y}P_hB8K5y6dl3+M{6+Si7kfZ&{uB&HLvV#(p>1jA9o}` z3?pOiqgrKs=jl6(-c^AH_z9#iS#&K4D32B8Vzq1_Dx_w#rq63{yDnN z`h>KVYAxQ0_BUBq_Ss3=KKJe}0_{QUqETfO09D%C!Tyf7upz9~Z=O^=R%Zp{Ppmte zk_Wj^>Pj!kQiZ%(3urohoWHs}9hn4ugPN7Xmkc8K4H|5yLT~> zj(uxiQmZufjPSts<})DJxLJ_U3yTif7p4+9V(nfyG1WLd!Bjw`x%UlV^}P$$aynt4 z)=71&g(nTk8wo2Poyjjb>8n1r@Mj#8a+2yuZmv`tre{`nla7-ZlCP_%$;dK^s4dqp zpKPJ~0*^8mW8mZD8{NW-ry@m}i4zJQMyXI|A0XOjqMTExgM0`;&UHkrxN@noO4~ zjXNbuY`V-E!$0QT8a>=>|)EaQ@4vS1^YlvcX}Qx*ffU!-PkWE?C|^gX`N3#FQ6(Q z+*Z0lJmgU_zax^!!C$whS~}SzGiNc&Vdq#hecLKczi=X`FF{)@O=YprwJ`rkNXF%2 z?sWRaztZ#h+xi)K$&J@1jUnFI0J)7Pw*q-2wPj6B$!^|-4NT_FrFGyBIfUjC_P8H@ zp~pWK0;<134BRt{Z3!xozCEN!c*4Rs{L(cY&p~~WBbmT|p$a7?V_oL5y+}nVGGP&x zP)v64m*K0o4HhxHHR0kj&@QURj4VBkNaUW>#AE^-s?cyrM~f*B?X8Pua&u2G`2vZq8Dolx1fTE_x$df^ zzvoU#e_z6|2el5*$~299i{Q0As-=FP0d6c>ANh1@Ox)D{h^tL5uHw>Z2Il$=^k#IT#1wRAr<)q-27_yYZ$>XMP$Q+kbs~c<5>#Dg$`7lFp-^4PG?kql z;h3bj>{8LzV=)0uuAG*3Nw^vvn{@G+Tj^S9mrA<`4ed;xT#iD%FstgFZXP-eba8u6 zbE|cWWQo42%Zxj@`Eiw>gD$V_Qx@c%Y7NU|a_N2G?XlLWqHt z47N3gfov=L-eW!(>4!QC6KYvpCg{kucR=;gD``52C zrIP~mZ%uF!Yg!w&ewkjb#e&+lc=|p+01;d98$8MZujGMz--yo=y-eX@2W3IuN7V+TZ7Vo%CP_|a#|6l*$(q+^W#GXrI#u^N zQ+=V_DeyBR?E3Jxsq!hS#5hTIBZf065@Oe0@s}?x>4apB5vlbJKu7fyi)hTSwtQ-d z;Vt+{<5jwQ==ZlDI&O04JSREklB-|Tmr^XX>xr_A!k9mh($5%JQ_)f+$J}&?2BCii zU<96haTDUncKBqDPr<+a`?iL5%p%9Y>`@2di`sPsvP?@_979S#_<|#=tovR?S)xy( zBn$Xmu8o=S9UIZjt^% z(h-?DN`R0+9p;<*25@(G0+w(BmMX!$7z?%HPg0Ke^GCblZ5iq-!+nu#{)qI3rGLmz z{fYUQR{!WEb>mdJV!FoUq|ZOMgc=!D1P&`ljE6=`Osw5}14Yl?p7eyT}pU${(S_Du$}#{O>D? zOxDGGAHFMtiM_`_>r%F|KLyg>4h z7Tv$kXjGk*-u!)5QJ|fxIDUhRd-uP;r`Y zUC>F$pZpi=!Rz`5_D^iouSMPYIt~u;>qWHl`v=trPAk9D(^po`=Fx$cG^E+k%WnWa zqOcs0hx>L}5ihSWdE`pNU-6n;y~pI_b$$6G1NA25-Vhci(`@4$lN6%Js>(N&JtzS${(aD4=jos?kYt(Q-$HY=Hg zBMj2?+l;c6ybDd5?XJh!E%Jxo01xy&+phbu8^aM&9`T4Y?loe1&w+l#w%+Fq&)Rt1 zKNkFJ&S$MalhaX-Fr3sEwMufODw?Mej*qMXNR5Yj2^H%FoM7SD_miOJC9EG_dR(R0 z+{Rny7%L7l#pe6nMR-3Hlh!D$^V+0O?}{ zMA7%5m)*MZ^=U(b1Bp@RemY!Mmj0nI&y0}Xg@dPDuN6Kd>)!1aIr-KNniCYdM6l)e z>Uc(SJmJa_BW&l|2b&otMHM&x)fdk?$u~gbrlCqL2XO{zq6_|A@RQo~SLoV#+K}lXcLP z(^o%peA(umQBz8=s@KK732;3E#9A#}BNurJF?1^6%&338-3GQbGQe;2LfV!xL;HX1 zhGCRv6ifD$a;J$q?fC?WEO!&sv+3*Ade6X{Rd*|;DEo988~G5BfzGWh&UFBR&Gap~ z6O8v}hm>66KN)XtX_A>mHRjp0xB27_WQeGKA!oO$U57qYB!jE0iLv@r;se`?lK^-^Fh zH7 zd@_U3mnWVft@7Z#fTq@iVo5xR`=I6uL;MeE2P4BR)-R*c^fbItO$&Lfmcl+I=8nLU zM#+WG<&mqA_4VXL)!LE0jc!@vBXwkqNYdnM{K+PMmAupqGoS7C#<)lWOaVTa2!iM5 zN5?_+-nxR>YQD|<;Rb%`>rp;(O*68T)3nyw&(UT3Be6XE_|I90f&_WDl);x&3v<+* z@uZEga^f^*LA=$@Xr^jSj{G1o0+*4Dc9DEJ9b%FVYgEB zucrBr^O`UYMkTU=NGAw%a_g>DYwLw7{*{%O?%ioz9D%IOls+6g;cMuhb#}M4TE+`f zjaXN4iA%4F*n;HZfY;Z&=jD~HX&PrSUZimZQxvuWx{;RGq2MsQkIsVYU5i^yiL~|a z$0F}19uzqUs+-JdXY?}01>M;|9H&Etrl<~%nGpETy1Jy=81t`H6QTEw+GFu9u+K}| z>0Uad{*o5ArmhY3w*@_tg4q)IJ9ffBXMok2Jr0^YvT0x}Wir@C1WO5%$pwFKJ@md; zC^XUmvF!*eXg8+s6?U9 zJ>TRDmUi*9yCNGu7MSfx_VPffSQ}YY9FfVLOY#+%aRYF;i7~$FOw^MC1($^79yMg# zpKv-JT!<1TAb%>uy0TGybCUzV^MLCSGz{|K!_3xXH9YHzkvUOJ&Uf@R%F~smkgLOL zS5+B=`y}CY7BWXCt1n6E-IQ3HUU?)wNK9!QEF%l1R||fDk%r=uvM7u+AHXm?*Tpc-uTP zv51t!OFk;V>WUxw)ja(zGsu7`vNkTZ1UifMJ8AFY_=4h~$Mvp%{beA6x)sOaPP}8i zSaX7W&qlru1mFK)G@9w;rg;OjYs91_r|)}+?egIr_-o&^pFbH1m%k7&aHmAOIO~?t zc>(=}JZ%T@ z?b;VoedA=h>g@U;J{A;^gL>SX8%E_=Sr8b0c=dZ1Htb@hBx@{L+lKdjf2EZrrj{Bi zq-|kbW)Yz=1Bjg?x1$#P251Av4Dsq^9od={F#>LyD}gap6V{7gu<0cLo8lCYTFf8v zm$p9)v`4>iIcmF*jsKwJot|!v3+>X*1R`?X!6xC6v9gzndQ>A1301Zxy7!*l;A$|Z zp#Ajd?HxZIPqlI>-ql+=FWFc1Th^`Mt#fngP@`pESigM=T@VmzkhO^Wm?iBKYO#39 zI?&NAolmAuq+B7W?Ngh-Z_dDrOHLp;!rbz8!CfS+$XStM^229MBx(W=+OVddFZMzx z>Sso53du;A*iwq*Wp}jNMwd0jEZNuwtzj+Lc+blSZJDSr!bTsLcn_UEr8098ge#l_ zFG}R6KF2T&mukt1I|u>*SIu@vCL0zldqrMaCDa1b>RqDadvZ!xV`*i6zQ1&I+YvH^ z4U{S9vId|#k$wq}Bq}E7RoX2j3Y}wiP>T^S`? zv*X2Swu^e&Oy3tW95ry#eq@par`!*mJ!B$lFAp1S{M!`02pJnLCIqePYO|vK%5bh7 zK{sR0iPHmLhCd&02E_~#uG_=zb>K`k25ciwuwWq0BQp##(>`c7yuWNboE?F}@m=8x zHb~84SG>2~7>bDa?wnfDgEj7biz!xMN_4EtsezN4oJzoo;E%2hAIhtR>8I}v8OMJ3 z19BD|fArdKU>sf+d*fv|7;&hXIn8CD6mB?o#P?;uF&Bt+QXzG6wbt)JR_WsIJHaPS zvx3G?{;>%@=>tdV3cycd$DdV3SF%d?SFY-z@~y9C7S=$iBD*nO^SOXOAr?zn+C)zu?j8lY^zs62;}Vgf`(HflB_8q_-a43gA~E(%I!_^2B0vp z)VZU9d~6axX~KR3G=Cl)t(zlkdTf7{Obn{I>eFOP9FVXdRU#2vPNvz)W8lFYCRkDE z_9rWdA`@#m+5=K;GG7S;D-)$h2Z^s`mL*o@hb80>Ln`gcXv|b+6*&eV=JQPCsn4*# zS|7@dtI?BBsv2At45vs2jk+C+J`wFZ&}Oj{SDm`?^^e%OsC05zCp|@`fa6 z%ksD!*fJ>OF(}f&mObinqMpq`(G5Qz7b4nx>>dmq)uL2|B(w%CGYKMBRXQe{@c7Wp z*n$a1Rt=B`1zg4$nFOw>I(Sy8^fZxqh39W`gqhv|f$}fP%mmh>1~8lrBW@V}c8}Uu z?cz`O2fW@;Tp)@E1}yr(_T2DhK@X4b<3CLn*Q0S99`F91b)YLC4~(E&8TrU}lFjhX zwZMiJDz$%$#r-^@@#mf#$^b6piW?zESX;ya?J3>es=Tz~T&V#tXqDHd8(Y0Ox4ocY zS%BWz#UhY|QFkVhK{(ckE!%R6|7a&$Qa|_UX2M%pP0z+@`i0tMh)(RLcT@MhD9$|k zRSC=uEu3pI6;&(wfUlb5Q@5a(7O$R$%3{0UQ6~9`KoFGB(g^b1zTBhW$N{>#!S*)r zw7D#0lE1)Jd1-Qs7@TdWOgv@~f8-w#d_DUxX6Dy++HBgSVv(J)s9QR$vj1TEH^G3o zhMCwC4p#pWOYJ@z?_pV##y~WGo02KN*pq5mr-zJSE`E-nrU%-%-1{UnchSb`wdgDK z$gcgiCPw@nA7K zhl=x~4jVZ=JKi^dp#F1KTrF!{`2`OOPNhhrB&Um8WASy11eA~@n)oYE@@Haj5vxJR zb?F!c1+3)x27vaF?Y@uk8IyZ}SN!v(&R$IjnGwl7*tuPf&tqbYoTZ`O>^juK^nw{r z{<`L9DzU5%Cd;jAGLc2mbcd8hFrGqS#u21|)o%S>A^wH427QQTSNyVER@d6LRSeRs zIyOxl<(jDHmQ|v5~sfnsz5PvVH2Vh}x1WDOA;b{1c zWrR^F;YI9DYT$lhvk-Q2wJ}uZKjUKg;XYB=Pe%fH*`2+{s)u4ZKZiUd%z6!FoMvf7m+pt3@i+6)CJG-#EDVa4_TRrOO zep^wAW1<|0u7*^94I&wv>vWu!{^DZwF)I|1;sWSL^Ql z1@J@_&Y6o2s|_xFGkrCl9Bc7A^tZ@H4;n{#8rNTLe^sn;(cmt>&Od2bOE)sy(ojS{ z5c@yima)S7Ie2B6Z$G|VVya!xVo;FG{5JMq$Xr$&TL3z@=$ep&96QrQKjL!t_sz)| zP`h|A_3xka>FlDzU+#*WLo z8NY6tc-|1qc%7(!UrQ}o1V_Rukjd#i#rlVl@ZZoCI`zOpn$aG=!8_(|f$V!X2^uzl zWn)t|)GQl!sVw}J6$0?4rkI}asR!~wfmd#KcpUYk$M~oNEkTQt)$1;CgiEjxRAvX$ z&+@7|XFt?YRXUv0T2$pZ#RXPwATBkvMrNsBuU-EN1vtEXYd{?86&6^owD$Fs-o-F- zqmI8VMyj)}55o5UcHiaCkwVQPm&nDz9@EsF;ml_o_pD*r&Kpn>)A4 zHPS9!p{Aw(7gD>a*}< z#?9~$f8uZx0A4vP$!flo$b%(@6=&M+|RmpF`SGMLrDqls-5S3yap>c4Z|9DAPx@upAx4D{8zpCvcry(}n z=~0!qb}iAg>j{f(VpZpx)pl(bXu328r;*&`CcyVX%PahK%|8q(lfYWljjM~|4@WpB zx)%|F$Wf(sUhI~{I3uGIK8K--?~NEK96l9Ay4Y8XN{+MlO`LINxNG~_gb0L50QpjV z26I2^2kDOh8nTICNK3=97~PCCP56mJ2%7)~#I?Lr>H`ZYbV&Iu$W^-Wb(7*i+z%&dzwM@d9#CLEz094u0_XTIC#s$l1;q))Gx`Y?d-aY?OEdDHy zWaNHTeDj&RseXDQO*6pmi=MVBj8Pp+_II7*El3 z3P*D#;ZMTO&zgPto$jO$KkWN->AI-9!FFt>`v+)eC_#@`s-;8ISYwhj zO&n(57}s5~qD*2R3Fj+Mz7LjnL8x6#@&V||jCn9O#OoECLC4)si%gMgy{Bf6Ltj2m zjOHNsu&?V-IL?GctZnf?eY+)if6)TQ>nwq@9k5aR7+iW^BJOCBh#elqp{7-kQ}{zE z>(^ZVYE^_X4h2PcOmrOY@##{mK1;BR4bs+sD?xqAV>x#|L}eV9g*> zK;;QOYVvZ~YNYEBLV209sX3`Ml$)!ysanQvH0K(vUlu!}N)aQDL`Uxbnj7OC+^TpG z6@|x9U`{CLg&OKgsU`)CR68vToPto;pS6n#x=%C7l zkeZb8ixN7>J8^PuQ7JJi43^Nz=6%!t_xhhR!=17b$1x->d{F|6#g8y1cD%ak3pPsk zTR)i;$mSEA>&0Gr^x!1}1b^w_Er3qsWCm{QtBQV$?TxHz9@wIPP+Yn_BTRGigW9tt zgg5T}Qv6|awfq%*|B8fzi$AIZs=qy8b#~9W ztwMcP*W7k4z_w5+*}{MqK7nKvX-?8^Ten)j$|OfqSvA3Fj_u-;UCoqyY#Xj%lQ|XC z(j#qKn7UXz)48s3g57o4pLWFhbiAm7jp>27(nEx%yNX$*Bq0(8{I9;fQoDeyjUX6_ z%bGZJuLCCj!Ehe%3;_6heRsGi6ZH~1`AOWUGU8mLCg&6-*(lVQ&@WTgqPhyOtg-p1 z^&M7;(jXawLyQx+>D{Bl1zB2m`q&aj!;%uF6tmG6XaS-ma57OJQ#Jxlv4}yH z^P#6h*4pyE8ci?PPk@o+MyJq;{5G5%!vRZdt+d-Pf!RFhcei%%NI;VMnLukj9v=BLW>3iG9m*$ggCb!ZgKW za%oLmAMsAI9nZfW6^Rt-DfRS&YyRz-45)Fm?F2;kThVy%`xx{j#lpDpx*kg}^iDe@2Vi54z;7e(TJ))o;*Z(^*<Jo zTV^ADX!~rU*_-|b$gdGv`mm_$AyODfT*Jlu+orA)Juv=U`0uGH2RBO_7Y3vD^O?&) zI(0<3J_r|NQA#``Yn~zBNR2Vb-v9mu6nkcSBQ;8_X9<*Ph(bUZDcxf&kZmQLg{!g(cJQO;XQ~ln-sJRa$35B`n+V`f)1kc*3&js3ly#HSSYA2Q0 zx{sXpldsub)kh7Pgbt(=p?+tOrjcM}3iL0G*mSWhc-#9|zLl|(7fT_PneAZ2sJ^6+ z_!Q|uHsmW1`B)Dy79^2={sY}bfqa{0+Q#~ym;O|+po3y8g|-OqQFHCEtb)Pu5pQSF z8^Xr?4DFdbl{4GH5@Dp^{$B91S!v6pY>(-#<*zyl5TSpdCXKq>nU@C;50_In{ z9}3*DrbtJUSn=LTk~QMi>$f5O1qQ;9M&UT?ewDJuEIoro0+y(IF?4bC+nEG`-oR>o z)a{6aV2;Yd{A@4m1e%j$qzMGxKCS@kZp+o0CYb*KV$OVg_`rUa`BAuw4MoR0a5c@c zvWwB33vHO~^T(SJz8tK28}PE?D-mpiJRr3lMj+NXrOK> zRLM64a$`hR!3C>3GbDi|#l}N&pbx}P06CLQM`wUEkfUHmn_ED6QELF4lTJvF%O>vR85@}c|;RGJ)iBLS8IK{{o?X4ygXX60B z8hVJzzG|DY-$ACeR7S$gd{XYl+Ht)wutA-PW#HNoYe<4sJlZ^2tQ$<-{ zXw;tTSAA9v8*(Syq$P@T9w+vG8gxmpWkRaA8nNNqZWKT=z=NPoIMv~Zops`Ee{D|E z%2xonv6%3~on)E%RQ~{n`s+r#g4`JWzeKCjG>S3Qf4#VU^d?b~B%Abc<=0r_WC#1( z=KJXuS{dOAlJ+0rx5ByzA%qh3NC-A@VjGYnf%uA(SSp#*Ju|!GWNq&jSp7NCQzh5t ziI-HC-`I3+)vhZjXH{Mc2GyC7hd=v#q$IJ6L(*=ngp<;}sN7KdsTvruVeTiuR^St6 z9Kqp9WMsxJ(XtW_q-rS=8HiNfW#{7M?WcS(N{5w_duzv(da!Pbkfd>;D8VTnK^NAt zvNRhQ@&LcWmdy-+GoqeAAU6hMV{gheHEm5*c?o4M1e?ONFqr!K*+I7}c71}o^fch+ z*(yqDyTc;`Vy!)xu+ii>E5P>97CQ%@Ct3OJP?HBmftHC>R4J$_3EY0#L zlGpzLlvP&*Um-Cq*9>0PcGtKOSSy5MD@8pf&RAG+(_7F+XQM`w3S?7%(qgynsjM1ddVZ4J3!fuhY)z@e7X2Jy(AT*Lf7EBm-LPRyb$`yLu@TV8Ae#a<9C`B`JQ z`>JgeSQh$aSmnlzqaX%PXc~RB$#bfQNZXUB1S5wpZA9rvvF)gA2ee+Z$Of^)rTO;M z?1oGqDh1SzT&iQHS9fHx7Ps@NR9h7MkkU+40?yhhC@fE~3bG>bKiPc&>U_SNVH{{* zR+}J%($~0NJSw%(WV^kA3D8rls zev%ZP)5QI>)a46)1AFbF%h!R0@vvUBzW&4`9!Jq5Z;^#vIPYL1a9*QqdaB zLisrj6D)c;Z{tb2@P^iH2(mbh`Ts8 zd#Qe>Ec&mq;_RNI+_u~oX(Ggq4P4!ex`uK7UaMC{byk*yP3?E2vZ3j0Tv@$Mw`1jh z_^l{1kL_QTMf-SHPe(sv+=ax*^)6mU-d7tJ35kZ_yggtL@&yOC!nXF%N)TI?Bg)K| zKO$_Yk6f+^Bxnkt7SglA0?QSPCrh7|K_v6-s-kul7bR_b-3a(qhzzm8@^KhzdAd)v z)LZVQ9dvD-ZyS81YC+_0V?i3kcuuj-oqi=x2~n;1h9C0NbOp#XSi%hE#b&riDvUT= zi1>=nDnOJm_E9Mc$hQ(N&Zn8=-{u+`m5QcWy$`#tJ`Q3fntX_Y;;(Cg^DyKBwzME= z9AtWPBWHXZ?lEQo9ItCyKrRpYg>!CQomy=rw`^Sl{%(AQayE6;e}oEJ+J0gDYbG;8 zgp_oPjV-5Vx`3n^1Qk8htqhola>kb-*B$}aEX*T`cwL;>3VJDUfo77HUGcp)vieJJ zP?0Q|kKxG7a2i=8BVLoqLHR(hVw#m!4-01Jcam@2=49h|Q=pJRW59OlR3@3nex}3w zXftM(Ld}mNLkeGBxnAr5>|0sOa~e63rA=4Dh?cebM`yNx>;>`GI8gB<=nueN zh@gc+hJL_LcqH_S#v805Hq2Bcbpaz0-sh)Jg-%LF0rvo@xX02Df*yrY$dSx@>8q42 zxhBnxV#T`?`bq<+*1!voi)%x(bRx@AHt;R@uHNnx)%}Eg07MPbyIP|ri6pRY8XMcY zN&^RAGlxgLu0N$iVl759!QQj*a&HqI4_e%Fy@>CysX8W#s}^0K)zBQWu7w$!#DnDo z4GQz&!limXN>#27kF)N$TZSQYky*eX*n@I7>BL^824L}HMh$J!rBqZ_Z;0d<_ zrF1WYo&1_6>D9Lp_@ui?Mw;H_dnmP#CxSdsWW$m`esz#9u-H=FWHjIwc|4*9fFBOM zg7>OdEK58JRzO)KRTml<&jY28-&P-@-@CKDB8qN{)4)^IeHw}txBDRr|@i65g>Wn?~`!!8O z)w(QmU7IRc-SOugGr~$dXY;AC735hS#a;`36`D09{{T+oNK!&Uby62W$Pep6K*8pm zmDi7NI?N~^Y>=N)Z5>}H?&H0wf5mv;P3r{{W)@0P4^f$JgQ!$Qz)0Yujk> zQ!yM?jP(uzpjblK6txD+W)XsSp+W7IfYZ8QY zFgISH2M>+uQ0AHVD2WxvQCJn*v~`pA(^~>pB7BT;urLD8kmBFrQ>hfPGedZBgO~>* z5QXhk1+@0uRAjP3wPQBsCSmh+5(hp?DeB=9e4B3W5*CekNEG*Jr7{JDyTRy)D9{iu z{#CucCXH4^zXHhAE8DP;KFd`}qd7JX$T^(-MbXJUQB#Hn)tiz42 zEqh#%r*%OVWO(h4?6NFUthWQeFVk+}$GVRgVCNyzPn(Lry2E#hWnk7JlHlvYlCdzy zw8WMVOZ`Q0GQql8K_b&~_O-tHP5M6<0N1xUE&*&S=Aib=FQUB_nd)5PicC9vB!2HG za=xJ#wU60d{-37yzlx)q+4dH-{GIb^+Hf|^7_e|*5aYxws5qbQW6h5woj8ctX7-U8PoK1 z`O?d94af|1zkgME)jdv%zWW@RuQ1U0feN169*{5D+f}WyM%BSr%s~qnSCR6pc%5v< ztz5Dez~gF;_}*DQ5=3glwCWGWlKEvd@Fv}F9O5?Q4f`%B^>9yUv7;g}t?hC>yngz* z5v~>A5qX;tuO6U3jYiy+uLCR)xmZeu>m`cx-?07Q>M5QXmQcDd%FPRC%HAGhO%KFa z(H2Sk3duMhrmfK;b_1uvvSw=pZ0!V+Kn>N6(%pxJYh?nsI#l8&A}76gxlTUDW^Pi6>bP+SmbL0S6gthip&U0x)3Cd2gwlCPiq?*njMTKonVOMustakLXH+B z059(~6y&SGrw%oQN0TRrCxF=MEpvbKXazqa8g3{{6CIs;U>(?BC zSsSw5JnD9QGL>BuZoZE6_WXMGrWo0P4ya1>H(AtpO`87dwOJb?z=+JsiZ0ATsP87i zzFk$k9^9WxdShV6*tR@fx*!RO0h#S$^0uzW-(O$r>00Qjx;}4y*Lyn+daZluSAr5~ zz}s7E_xI6*NfgkDF`W+*t*>U5pwW{)M1>}AQ*SsVb-px@JPpXj*V0h?2)M60J3Uoy zggNm?2I17!@|2kt{0f8AN4A9?O>|OSMETR1F1}i`mgjai*1DGmBjjhX%4Eizsp%`_ zG9JLD#*+9ct>6dLY2|KmBOB{=U(3`q{NZfXioKe@^^Hp6zOT z5nXI%i6j%V3zT~}<)|JnWyNanX0m%OA?68CT2W#xuU8VRJBZ^&Fj*0)jPf2-$AASq`%MNE37Z6k(M8KfLdXXYZ)?#^R*r}Ct~ry|5Heg!u*sn27_F#LWNZGX*BdNq zm~r1-JV)DIXAv%L*%aViLtjP=0k}P*odBkAk6Udp`g4Ag7z)LQHKc|+*hX9MoAM-h zYs=rO#@OKX{e9k5&mg#)#lQi&t;GT7T+W9|xXujUHq1_kk)(7PTqaoJIPPR{v6Znb zK(OxZ_EAZh*&SnR%fsCLJ%yEs!hA`gQ5=iGl}AnfBEEreeB1rDqZgQ5u za4tV_A-6CRGk?C-J!>QEcm;9^Fhn|#G;&_|*YQ(8vT9q*mNqu(R%;Gcw@CZx(65m` z@WWekx5TYI30#XBBl>#S5J^0M?KJjc)ex8J%WDk)=>Yf8d#ulhcqlz_x{>0e%a!5ILb<4h@wi|g^MW5sT^BSLIP zD;mK*V)YgzJ4e;X(;-UZZfiahNCuruSx6S_YZ~FQ%SWMGVKSHMOlFEgG-C^!`%a8K zhluvq8gSt2YE+e6Ew(I;`}Xp!p_msZkqRt;{{WkfcWQEuEvf-_bunZ-5(`Lt$o1jW ztXJ<^U4hg`>M&&ewCCrsM}E>SMd<2$pl#`hETo%j)TOQWin##2wpB+5u!F5@BUP3o z8%r4qFEUxzw68zAT9*Y_IdJR8#)is!xQ~S%z`<17QU#DPAx|rfYwV}yNh*%#D1rd)8jX4|nsOBd#jV{-T7)e%t3P`4#&;U6zcE?y>Vur!h@ z!o?VM5PUjZ*xH&ArJF9{npJO=IM-9=IX3vw@m~P)e0YbFRyW8$&vCfu)pqR zjopg%1Je*8iO{j5RHPPe53?2=OdL3|F_9t1fkHzXizu-5iw?4pcR zePOlqO}6d|gqC@B>CYPs)dm+QkW7eU_|UT8eQ%U=e_6<5pBHi|c;t49Wf}n%VwsSi zW0_Xf4URH&>r!&8Zm6D2MIBy5o(u7?JvGYsg8hfCFbrM(X$6TLy(Fpl@uxyICYczd zK1WJKZ6qQ{Px&jWt-Exrr);i z5#-V5&+`I!re=M4{gPCK?yl} zv5nGOX&VQ(_((s(zEwDKeFrDg_}LNWVL_CHbWajUq-s2B4wd!l(xmx|C~Xp}a8fvM zrb4q4DC5Y}0KpeRp!+B=oFD%HAogd7MZRLCrNNp!z{7|PLP|ZiRchwkvnO7QN;#*h z^08;h!mqhB+p;75(yN2yl&djtFr^RAS0jN<;?0r=1X$unS)zM}3fBy` zRIiu9-ujE=q~Q7I>wl((#p{W@lqrpx3&u}2Q_|M^DVoTs;Gr*2Q{vG$3dkqYU-*}? zg1f%;A-QQ1C2~1Nw?F!xp1g}==vTuiI@xU9u+ea#brxn)E;PF)ppSLMUh4!X z@+XUJZ0)VW03r@KbypI@ zvd&CkWf0_&190jZasCFc;z?NTUrfHDs_hH3=e4HB@Z-I~zfZ^yJKKkCF9ULPzoy=U zUDelk+~U638UFyag}>`MpO}xxr{i8vq3QX4cH^Va_Whq9#d3B4Is--k3I~Nog583> z)OhWYmei1MZ?cFD)r$@{>@>oj_g=Azuhcz?9pBTlK_cBaO4b(o+IMrJ{*Kg7#$(d-Ca1TQ} zc68Ff(lW8 zFG%IoE&=ymm$AA&jr2H+Xx^hj17dEt-7=s4)c+&m4a;5<(xd)IPv`j%TS3iG2G6l|#A=Kfb1VxgF zK4w)t4xfo_%nD1^O6#;>`PbKRNsGTOb%1=NjpSt3Hf1(ujGijodq)Fae#MizoENR@ z5eCtY>(WO+kx1-ix%=w({YX*%La#?gnHbW*@!Un=EI9ZH=|9e=>%0%_EQvi&w{%YK zi^Z0r#4<2waOOSzHZB zWmW$Gmesoq7{gEylk)k8r@q3}EE=G=EflWDil}e5`q5@ab0afn#Tt&FuhvKQ02}P2 zm_dnR?8(geZIf+ZQhRw-D_Bq^oVhkg7%kLw1E!ossgo6=Y@+e)UkCd4ap zsU^7;BL40bR6@ZRxl0Sey*j&eJ-fvdNc$3QsHP5je5MHOd2&`g;2M?DYOyZokJ089 z0W5mJ><7ZYdmCERV%-lt9Ag{C1cy-^@p++!fHMU@eICr74)f}~`F1}`=5DZ{X(NUd z$LbdeD^3zOU_S`GS>mLs@-X^cy84qlZODY@|3k>~@WSUlsVC%CQ26zrbVldrQNqQs=!%{&gqF>w=g60FyC^KAVYwxQ{Cs zi+1%g{{R}L#IGmHsT0V7S5}dNFSU(_-%2_91Lz&I{j<^gW(p_*2of>uRRW!GOm>FS zwwEJI_t7&qHs^0;V-Y_`-fiD%CIQWfWJc**^$S>O@9v_EaF4H^jDdDP_iR|YXUCj= z?m?om%q@0bm#8ooVaB<=nkY=__3z3m-i|w#{prVoT+)Brn8tC!-av0-Ci!Ib@n*;)b?(colFjM4%kl0caj`GU59`;8@=Tnv(D z=yj^+If(~^cHXc7<~A2P9!8i14RU+G<4VRShEBBriy!4!#D$w%)-WFDfsL7t(M*Am zf(^y=Za(8e11AFdsq}2ahBuE-CzrTkxn_w!>qEq-9GnaLMRV=b8An#V(&3!qc_Pgi zZpgxDl4#y0)D-Z&*S6Y$?5TOlZw92S40VOP(eqt_L!l^5^{mvRCk)PXZ8QQxpw9=) zc{FEG!@9>&N|O3houtNE&>+zHNmGA3kK67N{6rh zx^zEbkwIIY-V~NAP;KTT(_%boE^FN{S-!Ij>E+P`Wd9gLXxd9ufoqH`y5;Vq7zwj~A z3HuGO%T^$HWCphj-ufDi{{UuQydu9BRGDt~zd%q4 z<8TSJRd7qQOqj`kDRl=XrEFw?oQ~^{MP&E1B3KiTXEr5AYrvzk8J>g&Q0!gJ)P9TkVbB9IMy*J z9WB}YYZxZ{PcBxhV@`{M%d~$A#sJ(flOiTp`AM+?iZR6gr+q$y8VPrOzoHh5nKH4! z3aw*hkuAZ=Hefz)ZFBC`k0f;H-P1VsW*kUXUz$Q};fAQd{D-#VopSNP)WwM-E<(cW z5Qi&{&TOu&ua;YvKA!z3y7mP60^Tr`N(k^|XyI_A5gehk<94%Ac}7DG$-10KhUL_)&cB3%_f(NvaC>Tn)qLoO%PREq zxA>}!;GI$5zL|QX8~3RWZgA;bK!}opimtEnju!BpD%$K%V)!}?9-`WD=a`(uc!-W( zyY&!tQN!l>X`PUsESg_byz0^IkTkLz1)4$7i+y;S9)AGv`8M?C#Q3Pskj8wOou!^* z#It|L3KXa;&U60&l!Qn0`J(E#UQR!T-Hd9M5pr<<0HB*$+&-vn3({CXC7j#)R>J+> zDhUSMA8B&E5~%=i{bSM^uJY3>5Z)M~3~>ER#dl%@r-*>5r%7+a$uk$B%A%-aLrb z#U&b}n`+iPM`d$gN@eM}p%U-OjHhhhl0Bz&Ju8(I!qG|Ot&Gqgm7|ndv^NJsK{(_s zPEp18Y2uZ$pQs5XoCYKE0imvR=2_Ckn`tW)^QDO-^p+MHct;wq@JSlAmN}%hr=<{` zO@P0|@u18I!GCgkO@U<#{{Z!i)2;-3lT9!kCA(Dg^SQr{SSTXiJJ?wB;aJCw>_|4!^^8R#ewQnOaahMT z^~$#EyyEP5p_oSEe1#eY7P>1QN-ugxmj!8xwsZdgS^850A+j^F7-7P(vI*h=L$qU0 z58@UUJ+!+uD`b6pU2t&h^9+!6GMoWq9QD@!?AWc;*H>UiRqr1AbIIthyVja3 zloQvd%#5IXMf9uCh0c)%*|$WGDkAwlUZqFnH|5%D-UyeIEn9XhBN`Vm1!lF!AWpTs zQWe-ZI7PLrIa-)ghW`NNM8p38X2OJO*%4TdRwm%m*kn>a@k44e`pZ9XBg2P^sebty z{{a4))mawD)zV5pHocE`3Ue4&eF^O2SOyG@$nwja6UbAD%t6aql( z_P;%DM-f=Y3V~v8VX5sViG_J$7)-87Cf}&zPypvZDCS>KH#|u&rg@@@c8nj}!sr-| zU0Q=~?$cb`S<|T;Pkq9UEKF=P3NvS8K0rKGE#cwvfq#8LRuOlQ4=&M11Yl|hQ11S8 zQv*QTfK`ouCCLMw{{Z@_kgN_h>y$HaPaOa&yQ=GX<@q~+OgYHe5dctA;{c!o{<}S&0`QgmL9W37 z1OW02SqcDn{*L744MzdJbWdBq#6Fzxx~x$KV)78x_`0wxk@w0|09*oL`#QG(bLC)PK1Y#8^p&aLMJND zV{dC`3$nB0q7&c~;OFHN;^i0O=HrtP6qgVZqWgDacoFAg?gaYcSn zaiFMxs4ySDpqRX%xG?ZP*#C)n>5a$U*>*VQ0 zr>OX1X?{^UEnASY$3L`xO!I$7s^s9~9N=IN^zn43``6q`IR7s?0ELwl6$M`u1q$-L z=r1a-EH5mkpd=_HETkwTDk#kGZ>;_Qn6Lkg{oh#m|4%IMi!!|bIL7~Ql>d-kyzigO z|ML5noBvul4jwPd#ph*RJf8z_{^jie`=OWhhw!`(hykD?BO{|AqoSaoVxYeKu+ULa z(Xn1(Vq#%pzQTL;uZH&u2Nw?)=M^C#5fLFF6(uDl743gD1T-`>>{r-icz9$K#Q4M% z|MlSi@5S>F03RJe4nYnH0Uv;fkAQ@a@H_@61pp9{5D*al`}Kj0f{28QfcEn8_=o;K z9v~nh0gzEp@c@VjNJxk%h%b!DC`c%Pf9Uwg^!xW zzefrBWGONc|6(TTnAeXMdXbHYfDAxIeECE9msA9NM5Grqviv`26mNk@#?1EbzbX{; z1fSOcuaFR4ltsb^$O3q%ic?Q^&5`qdBkQx^nNG1zmt{p}s&s$e_G{{3GBl9k3R>iK zcYdQCp zwzV~}j}^e0s;>@+lcMUR-%>Pg-c!-(dDGKg)6=HC(Y;wr7kdih&8!mFil|eRwirRY~i#}gI^fqYisv^Q!>!z;0oJ0=w+64|E7cl#? zNI)E+szLb`!CvqoOd$(qKWaCNd&F`rt;6)k#@x2%4R{`dOZlC~hgf0W?GYb#|Mo_S zSTx{GJ5iz>&}*+s1-w+Gh*EIR(sO2sp*lj|>ufQV%2WiOmwug_BkXInlwDi#%QWwY z%jvKhN|SMx+7-A}I|+aG4V{1ucdy7bRd*hWoT);2+mQyq3HKHNjYUmt>#-NyYX=bxY;oXr~#<6Z}{A)p>n128*M45)U_qG{617@_m2OFji;*3OTB< zR3>9Vpt9S+jZpn7gG&UX20pIK2y8<-%nqt-EkuE0N+Y7Ctcg)iOsx@Z$%A7Qsg)x}av-lm>^pdt^kMMVC;`YaYRRI)v6#y>AIX zL$qlaz^u;P2TMU}BM&qVtnCxaY!gzIU;Zgy=|#qBXpX7Pf-pV3Wme-Npg{H5eBHPd zO>cJ~(fvpNqjOqx#W~_yQEodKz2Kt9coU#kXppSQazk<^o1L3|ay0!4@G6TY+7j)m z6;!`q@^|TPB*B|6Z^zX%gG^u_o3Nq4(Tad4hW_ASRD72}%_*MpkENVLD74p<`Wiwg zT=!He!pVidp#!x`wOUblRI9Q*%ycG2TJJ|%D02na^D%XYdZ>YdjknE=XhQK%KkVag$B;%-KK>zvRgzd*xkl zcAQ$dvjqJK3O!MC?es=F|2DuH>0`__{pCKl5Tt}DEdn4^{%Wdjv$r5bOYmu$>{EOW zW|YRRs=Q%nm@AU|Z7XkQWUCK$>IV;v%#hzc{PsSXh>$5qXH=MmZUdEd#-nRx4&Q)t zpMN7q1XWt~S{D|IrfF&%JXN(0C8Ek~2y;`z-EfEO)!CPKM=f2{&8P$Oncla=({m89 zW?UZLg=S-Fl^yfHR zlaOKxx5OIgQmJ7K#=4&B6t!dhSo-)-51LYpS{7bTBy^*e^qt;*V(FtK!D2f5u1X}% zC;|8}L9cP9nUnCf_?-N`LGfA>7@4_@X-YdoSM_D_nR%x}KT)!RH?XB1kKrd~n+fyA zGEPYAs55FzqRRS>VG@|flTls1{g`n5;;c@7=h=$%%L(nFXFy=G$YVG~Qw(u`a%E&X z#kcW%IgAhAU8qJ;R(@;v2k$vkx9ACp^ip07?&~h55)Em7EYw79d1XnjXpi_L33)uu zKHx&JcT`S9Lm%Sc0$0srl|(%Q^i4N5mi~XD7p8NUVFF@Nng;1NBLa+6mQx5N zdq>BjPsDyXgg%-dQ}kDICluN|oW|0%m$;$zJiSh%cUv*yt3qvpUFLD zq-vzg1fN4}+enpAq}y=Z9=fXS$dSZN7Gj4UlYT}2oNv0w~h1?x((Lfo%K8{ z>vy5h1qt`sXUZBNhZ)#U#8iI(84i+ju-@Fx2*3LS0O zz|{qnORdmuFZ^6x9Hyc~o5#NoUzh=8Om*bDtC~mdFTGe;{gfy{O)52$Na>r=*d4*d zX*hy;UPN@3y=d1(g;XuhM%sgkRg@PAVG0qj9l_lM0d8WpC2zRD%|)wqI1^{t)L1UN7(mE&0;O=dQ&^Yc`rsgz|Ow3ao zuZ8H4EaSfy z@W;ZMBO9WH3AEVksYEhTyWkP2*)!#%O0ncNsu^$V2tuSR>R=f!#IzqitB%CA7*H7-v$F8t0}1xHg%tt)U-&fxqDz zVQ}s5-HHHA-RHM1!P4r8yGk0|VD}LFAf@?8=@^2l& znV&8K=;AzriPUZixd7a@!HXK(sMa<01@?BPo`s=E7uekE8^YQc6ZN)BrV(BlSLGM4%l+J|M)4{2?Gt*IQhBfr){-< zfx^bZZhJAFx$Rv{0RWQ@nz_%l1HQT8mt(%xbYCTN&Nv zNNcL9u8s%9Sgu=}9gBvcHJ&QM7IAavr{xKSs(fOiA;OUbb;YqRkE*V%l?(WMXg2p9 z$94T|D3azZOA?QkpRO9cvENyiayL;zM9d9z~H9#>hM$k&> zqCFG7s{!LA5BIv0SG^~Cqn*+Si#|Z@*ZWQWo#Si^H-E#3trNe3?ojy|pmXT-n{-D= z_6aL?a#MBHf${M5Qsy34n-hTyZGznrflzzTmKMf`SPwT@gfQAk`V{mi3jnuQu!SFu zK`_?E&?nkmvK@jBI){~S0kprHqK9z%9|e(kY1eLoN~Pq?&AtiEr-4Z--=sWN?TWvv z;+~senq-wzo{hEY7W=7cr*mo73MHg<=;}Mxx#Mg2924E|>1Wig(=fE*eU)5tE>zcC zZ4@#&dbtNX8!jBZ5a~Nbiwz2YV`yG?fUvi)+Qj+EMRU9`^;1X)8F9P|bDHIrk1)T)3JMtX%AdR1_*eS11qOSEQ2!XIu92vvw3a_$N@+2H#`nEcHambB zJzkmaOAib5s4dJG2}6M8Rt26Cnnz8;mGox7Cz@Mzib#9aT0tO?$7 zw?5HSwD@__+ltdPREUu&y%agMp(A2AJ9ws4#tnIU)=jVIS-7~a@@<;Js2#i3b<2W5#`Z5_; ztN7);eMgRvf>%L1CMgZw&O)LhzB?F4R+Wm~+@<IGz^zWOGXGatxcXJsUd zwrJ%oiLzHf(vHJ&)y2h6%isO7*qfX&+!LFsEJJ*YtDf(Zw(_2!{ph`-OJHkZ4YG<7 zQb-d{#)qCXy3L|0`J7KBK$S$O|87$p;;)Ws9xnHNLkJNnwl=CW8-lFa_o);-e9q{B ziC$hEzF<3+agml_I4}T-p&Rnmh$*)78&}BUaU|Vq(#8cX*w9xC*y~t>qX3+kJP2nO zJ*vqsA5m!zs+x480E1nKSu|{rkI+W6wgV0jC?s-jHcP;!eTmc^h1}IxjwQ|x$)e|L z#^%I2Vmp#lOvsR!dDi9y0vxRmE<)dxeM4^wj(xS*mqj6vlf~O zJ6vYek0Na*O*qTj7u?xkwS+&&fGRg()r)K>eB^grQ-tGCl%D<*ZPXFF~ zQaSDNIUr<-F_A%qq!Xb0QRe!$XtimDjvY(Bq+XXESZkBo(5#GFg+Gfuo38z$N(Y?Ji=r-!AnN^6*Up@IzuxZN6;eN*PtC_wrxlq-uX?=>wA!32C2O zq@J*X_XXw0TrCY85kP^MLM=)4fZOh5^O3A61~;d3B8z1nV8jYh3VCDR60@!7Ec!Fx z;;0J0D@)BL62T5PHP-*{n`eM&2IEM7CP=pH9rvG!n@7$=4@(N1*`!|+DMY`0P`=22 zFf;g}kV#Rx{|kZ<*CYW(`vv}RE(wO#ruw^cq25&Yb1PYZ)d1L1E?gZL5hjuE!ei|n z@d+#_U7LrGu50Do=y>v>68L!x$yRVVzepSRft&IXeU4;jD5tz;v)ueMAXGx0@?-iHtIZVx@pn zSUCUS9}{;=lQUKt_2GkiZ!&@GIub~$Yp}7;%I=$*0B8F$G#VphK{h7B@Lg;1H4Be9 z0t)p<(0uPw)l~6MTz_}hMryg$Ql(y&zRYZjn;QxwHD8lCyA_@i*StNhFGZ5jV>g!- zQ6g6hQ(gHcJvI$PegUoqy9zC{+c`OcCD9g={akQ@DtPhuf+Kl%hIu4?qV57!5qT|@ zM~hfa)eW6uydeP7_F^oB+Y`zGg3$zsw>QoY5lUH@9rSa|2#mlC0ebfM@j{#1t+I^x z(funYdWi%>FUcjwXF%J&H%dz${cCF6MB>G-t+xhBK{-8cK8CT$q@=$9cgqWVDq32B z?qjBO$S&KlI&V>Pi7SG#7oG%WX^=r<7QxHhTRm6&yv~(l8GShBTV}w$1|7a+@0#YG z3PutVL(gWLxSCbirTmmPr}MiECRJP3=3c9-?aMT?W^2>M@dh#s`}M% z{SycMM|HiFIdp`;Zq&|0Xx;3(9f~V(qW|uqTJ}+fopUUA;Fi|q*F)~p1luU<Ch z(^$tzdp9V7n zySwF$jW>%O-IlAKv~tuT`A=7?7!>MH`a_$k=44;qs8`;!oVv|a5xn76VHs(k!3jdL z5Cu0Q?(ZcR8#QBeA(iFPcI`D%BIXUoo>bMT&FT`se|M(IdfRXzpe3GIYx!B#j<$mk zsNTL)URj|T;I;_tqAob$^Zd1|`aMk;8Nf9tZ(-C`n(K=08W*%-)q(z*nTtoXb_Yj! z!*AL1?rTIXWilD!59?-%sNtkfuGD`P3Gu7WMI^$kDHyWY0)JK75g@qILODSU-E5_` zBD(PTIJwUL8&oFykKhRBskEZ6nyDM)9!I~4JY0!j7GY^hVUqX;n$dP-GNzfRfpn@r zs@t>FBjK(3KVDA>gNv$MS06shdM9n08Pe2TO5*Qe`#4k!R#8`XN>XVcDJ{`Q>TzJU z>Gnzy?gy21&*T&F?t2-OapPm4Z)#v`f+X;P&HZA4x8hn2vO*3$Cq0njJ^j3r-$e8>PW#?M~fg6IT zqzPuXME8{aN4)$4F>kRXl2_(Ro|GP8xl6~HpFV3a-sh20iUW|2L6AqOr$tfSCHWXy zo->&YB05DGdkI&$Mq^{&)X+1ly&B4{O^;C zmR%n~*~wACbk|gy-?P`~$H#{asr*Y4dUk}i2mJSM`i<7&AxkGqv;Aa0Gi+K|gG&gX z;1zF@M^mcu%0j<3?i+k5Ik&sJ>`P)D-}7r`RHrkvg$^RK33kzl-YH$oO;MfMZ{?8 zRO<2}0t`P*F^2@#HIiynyIF%;nU9a0q)nEII2FEr&~-^}5*Ka=eg^nW6chHUXQhdX z?k-q*TFaX^8VX?+g-3q+!ai4=fgOL*qCGNqCGkanj)no?uGNR#{0!jO-wEWz3R5l- zk-?aV`(8&+KB%MHh%GYT*B)x{zPF4O{fnd;c_;U!{c2FQ!$sQ`%n5}?kC!C**8ure zf5zWWC>#MsJ~hS_9dkeSJktWnSs!dOM=nFm133IyKi9aL)Oa)KKECd|>BZX7U_b!X z?}r9ofu%Mb8skKSEn9mW9@y+fGEBsd!tvEu6@Yb^`a_f|0bV8HA(xC(;&+ z7P2Fvn}JEHF*Q0A+yl_kBDkc0Cre1@y{7eRYSeBTOHWHkU;XF9mE@lAK3dd~lJ?@> zVIGBIDY4sYxPEI6;$(~+Vb;# zVu!>wTq)b~h*4wu8CY7p+yQkhY=~TGu75-3Iw*j6dkv{4?^VDu`rZ`OE{ok2ZkAFV z0?US`Sty3bfpr1Ya6$aPa+tYu*rcA2_*CCE`eS6FWCC6wHZykN5qwD%J43K3 z8MM?+nS-Jelnq{4SS-pDq9wVzJHJ$)-+NjnphV5U(f;g@8FAEo(3(uqzc?c4Z3np_ zeg^bxVdayO`)u3h60tnpQp)0q*a*${^BKohh&SSBH#byIzZH3t{BgIjm@`fEg4KLc z&{%FeY%gSkNh)Aa>#gz!;_0-DmoVdX4#nOtIKa*ScZDlr7tZOobTYN+G5q?BukkfoTy=={N)IPLc4MZy`jw&!mR<^ zdLh(e-eQyIo!R9 zYK3-&7rb^I>M%XuG#*az{uzMc&;#FiO#kllHkpU2Dd!om3ynbb#P}}$o9AUmyjCyJ zbA%}h3Qu|T73?na(3}#mkwa`;p}Z?LIErLtKiZSNoQ{+J9G}BQ>~8CYVz$Usn0nrr z2eM6?r5cLkDl3}yw6!pGT8r_wam}mxnI4GYXc$kI&-2lwy;qb(sE1H1*tXd z*P7GDxMoyf3^yye^H3=sok}*sVyu$myt!teAc!s{2yPfpy!WBo6CweB#Y;%z49C~= z7hn|U;@#D+$U@5!!Ou3T$}=>eU}DyKS~M3E#2SgX>}`T)g9?JnNx)WZh!dq6uFYy< zd|c@POW-COXZa&#K4OfZ%y&nXO6?Z+Hq_UZ4cgd~6N_8B1y<|1f>VO3#)@8Uvbo&r zUm1^|DwDtG2~Fk&OTI1W-QAkZR40{aLv#JheiCXh;KH)HPyVgX>eHpzF5*Q~lAZgo z-doqj7UdAN?Jdfh?Q1GD-l5h5eg6vyV*yDMpp_)wEkI-%xM9B;8Jh)_LYLA_o2PeplzzVIKzsrYdxd=^%}*Ynxk}FS=ct5r!>pt9)-L z_*RtI&*S!PNwai=@eF@@BZk9tSY)OA3je0woZyk#B)b{p`#P3H!|rk_5%fO&5@pdd z)befG)q8ax+%t!m?q4Exm2lFPc$kf2bOA+uFJgAP7ItE!o!w|A0av-00vSErGMxRv z!23Sk2J&(!#=Qvp1!;bCd4$S;Z&Ue@fmZ9mSgwPj{dvhK|W?zVP5*_7~AVMXOYG@d`LAL@k1*^UR-cOK7)8Rk`+g@NNQ)4%78HU3+69Zve#wesH0UBc6on))THm(Ys} z&d_CLA-O#lNm5r-=ANaj85Iy89y0Q=i%|W1CQzwg9_2;wps!FKA$ zB!wa94NRy;3;T8=O+f{SGaY(rGvyD`uC8tDn&95;5I@~y)Mvoo)@h58mudu|tW%27bbUrpg-UX6FS^2al8ro1i$Y8~)%@x+3rHfYL& zbw{(vr&dR++soXqylevKHe2;@K_+b&`q5ZX(kj#7MwWem-i@2sN~kyb)`Jefd^TNP zBcQR8CJ|}^6k)m;WQrk>HHxZiwVO2%TsRCzuJUc0z_o`V8Amq;c; ze-zs9ZI58D&1-J?5QTH>e!f!u{qRuDlorZ!y$7j35E#w0tufAs&HNJTs?!0tH)mQ> z@O*6+Gjkh7bNwyBLDvz7jR1|BV|l9lj8f)nI#Fh7WBwkTKp-2Uu(m>R+`NyefR4&( z;jRZQCkT!{h&&p|T{fe3j-rZr$qm+ZC(y54B3-qFnvDNsN1AuGoa(5^WaT!2yE`K~ zOtDKlF_BDVhg2$btSs#$%PFy>t4|eJ<`cu0fg%W753acn;hFCpMKlkKWdG#DA#+Rd z;Y;?rc~W1G@nASSnoJ?PzIPPGO{M2E`nl^kK&JT&9ucdm&sjYcQyr#4bFIWf@rg1u z^IPhNDy6g9B;HP$@^GBuh5jH^i2Oz7g709Wj9&^=XGBahNlLj@*#NkmJ#mbiOGYdw zi%2nX)lk)qRF;7Nd0H%{0RMCk;K5bY(sh&0Twl{s(J-HG@|COKN^WBqqJdx;G@DAf zo!X3>T(DNuHv;#AK991Mk|0-Erd5KQVw@{C71JL4Dpx+Lk4;N6t$(hYqU(#eWzPXB zPYp~|gpxCgrqp`u+b&p-npv8mC06nf@R`J3fZ9lwO5aW-N_~OeB5Z`t+_N_U61JZK zLq$lp63eJ>KbFQPuS`wPMo!VA?z-$Orj{M%!&`yTg-9A^^(kh-@ze!D2C69Ao8Sn1;xpxf zKZ>>e^M#SGK_T5DjGSk`yJ>yKKPetZ8HNv?JaX1xr#-qg?|pH31`zD4K8)ujGl;&P zNcMWC`ML5jktMQZ4``u(_@g6JInmt!oraM_vHsBS8E|rBvJdjuV$LC4O2$Sj`0Zo< z(<4_DR0(!X&SY-V`sjQ-4$)Cc0}QT@hJ-0_Bt4}2sT#yA_65Pud~zobm@^$q-W`$U zxhwbS8e)OMW>lCiB=rqSZfIZ>+vcxnB^t%&P%Uiw$`;|6+Uh`6>u=t<^a8CvbbZ}L z>h}^jQ#;;UtmY>`BmWvsZ&0hL4h7O)q{_8HX)}{9ay1HuT>K>sN@f}`3Jd*2q}jzW zT#~&yz1wQCSvizb5sN#ig&XJR(t_1a+T!=D`8>9@dC1vQT?J3SJOdDPKXL7x2JCyc zY}%RZTTult^tc-p{xG>Oz*p8&6Oj2qXwWR@yG0e(Wr>Zz_YBBZ&xr$fYp={l*XF); zIE=l#QK&9<-L+)Lb;-D~q96O)U|j#jUh&pP$=zaBF80?gsUy19Pl)zB(#%0@qO?|M zZtse`jYE4~p&;(qo;H-jxB>7JZv)_(c32*2o>szPSX=HuBGs^=)>aUgqe)PNIfvLA zSUxj!Fwg~O8HT&IEH0P>u4Q8a6CU!N$Owz~zdP~vPySK;m^{Y_CrSq z-0cVVP0z^dk)Y`c#vOZL6k3m@3qsIvjny%gvbyZ`^{~o?P>5rpTEMDe;%z?qiHfY( zxH?38bDdG2p!~SmDKv=&qvBHi*|_g`YPb!V#Jn5biz83(j(f8bA%}GC zTCyL8#ATxhbqGYgNzqK2_&nqHpufAf(cV)nlBiy8j>R)kZ^PJ3P>hW1Kd26jru$7Q z4_~VQsY+lthVorq*C?1xfDV414W~pmrPdV?e~ar#XYJhv0pE20DLTA!pyEbRE^Sgy zRb;g=ibB3!Stf43VEQ`we%*4?Y!Qxu2eeym_U=jR_c`XwJ*Tf3jgx%Y3IYWyW2Hyf zGDznaR}D-H-HtL3c530=>565{swsy4P+y5*Ha1PKY|I=NE^^*gpQZj@Y5}7=dk^*# zkO+Qsav%h}8U?IRVFZt*G?^r=Z8lmsc8W3dR=H&Jq~N*hfJmZ&^2|ObszWPG{R?ot z7743{2+B3d7Hjv<@w{Yfs%evs`h8`L2AVBYEqQzC+Gn(UXR9ZNwp4Y$hYn^ieqGf5 zV4bu1q}jN3eR;vU)eeGq&}*h^7l8r7(qf_om>(|sk1#mWXS_vEOQ%R~!2!^j%Z$O((pA~nUOe=AE7X1l{| z{V?$iNada2Y#Zj69Qoszx9t`J-y~Tv%=m%}sq5ygmplqxtWRVx>hZH|H%Q}z7#gVf z+%x!DqSG*>f(y#%l>`tsJnOV1jJX?88iTnD=rs;UDrcq@xNNRO$}>_%9#7K)@p1Mf zamO&vPuzl`9~$`e$2>fEu>BjUv~niMY1Pez0py@dk(O80O++@H?JZ_)2Hj*Z5#bJL zw6+CuXboyFNmF`)$@z}(=6Jz?QE=gO$FgMMI28`N7%+Xl`wGtXy5|6e;QYT1UhjnTl(viUDSh1r~yn3wSA6 z4(VPuHb>#JV91A*L|`tXR(X}|>u0KpbiKkk@zpXyL^gCm6jmhQZvVA%O)N>F@atjA zcP#5-Pm-P_Xd}!|hVuuzD(Yo3;}F47g%`4{5Ovz>yS(RAZYI@583EX9I=No_~FI5X?`~j2E0`x4l&Pevwmblfeae37r3@2LkOTV2+s;>+`uA_Bz zHLvzt(1*rZiP;y^I^9i;b&AR4T}mdcgv_mj8aJN->G)-~ccTHPCb^f)C5y`6R=T17 zHJ?&hEBu3LGzr*#F0C&MPsnL5^Nd=Xuc1iconlH8c4D>X$tG-; zP#9`uA-Uyt=WHmV&tl`pV}q&fDp=_*j3j$Sx%68}4v)pr(QdnD2IzUse4Uwk2Hc{; z35coEsWdn)DRfuKDOy%zDZr2hi&5s@yG)2)F93^5gQQA*fh9uo{sYv0dyY9Lc@aN} zX)SA*omE{QX&PSHQMt3I-bvJdVk9R+;A~o0@*^ZQg8y{h>q`plCptEjrDCL8YHHVu zJ`w5``M^ovy+K7wapT1E9q)2KO+RCJMf+1Oi=SbU1z$GkVvwZ!s<>!JHsdXCcc54) zSDFP=`_}~0ftX0>OUS{p0Vcf3Ie!LFl@XGIEKaF}GO2vJmGYkf-J821DzuC5jz8zx zSOzRZiafqN?b^#xSZvO>PtRwyRX=90yQ7I>H2QnUx-`2Zw_e}VB)F~*Zt!{%($X@B zX`8XO(tq2DOpqyNC3*MpkW(Sj^ZK5-rz<>r|Nf=W+k^>|?h}mX3~w^crktb)E!$*$ z;zR$O!}U@aPU8}C;p!msxt4&#{uB+5DM&a`d^t)(&-LKJig zB0+TT{mHM<5K&t#?U1d@nUQ!30)#QX3Y}!~TqL6%f-hD~)lwDJlMQEd@T_maPONfY zl3%QY=57Qv1@ux6pm9v4E-(Y|rxoQkfa9~!e#yC}x8 zk3><8n>3XN8jZoR-1H=Yw*<85^>?G@%=!AgSyY(-qILjj5;kV;t1a~p62qeAb-l~8 zDco5i0Ow&hv|KFv`dsr1w!Z^FtxwN@IPMX**Gs)VZ?Yvh{=|XEw$oRVRfK2GSFuj& z-8w^c%%1wp&E@H^?dZ*%CC+qQ$82SNVY^sq)#ev90Wqty>rXMA47?vqn0Wb!J!gm2-J6fk{Q2zoIZzG*5!AHos&&eWZW9O7R8C? zt*xDFCnd{R@6>oA@%$Al>;npMM9bS=%*A-Egeb{%eZ5+RxKLgq9y!;SB>2a2dBve% zV+FfP-TJiHoWhc<04oR<2=9;7(!e!u${b#A3h>GXbdX!%s(@%j0Y9Al!jf0xh!)V4 zB6nhj!jKttb%Z%l5MvE=aibr}cmFwd5Fd*la7J$oxwE75JyVHg!OPULAp#|dg;%o-be zji&fr8M*!IeQ4&-KiM^U@+ghjW2W=BYgghZYQmplxR)qt`g9IPr%DLz#k)A`mwsqq z;>g1Wq%87Cb3ga37_2#>CsXS^m8fpDE_HzO-YB6|(-8M0oMLaXs@kEWQ&?p)3JCmN zE*MkWQe<33`{O>i+50dNT^c+F%Bn@MzTBL-_E-SwuHuFyF?MxCDT0 z1W=gZYnZ%rNkB5r0EhsBZpbTS4Wno()GmY)eMcz1$u6R9EyUo8X%2poDY8_RHKEMF z%(4VOm2!<-os!>jIkn&4*}cFn$yrJzvjkbgU&;$Pj|V;+FA0BPD83I4hc$U#rl9lp zckYgGQz(;J3!o&*n#sPg4hf5Izy7&SQ)?`P7NuN>5_^5wD$rd*zU4AeGbO27v8-YR zku)M#UsLH3oGea3qBWVDuh!3r8EJ~C@(}sOkR+c`fhTex^UZ0Ktr~g=A{ae+q#LtnG-@u>T93R zj_}lxePKx z#DuSGmLKy^xIzvayBx7aJ~S@V*)yUB=W86zUj@S!;epAybdbY7HTyW70}Hcjv=}4Y zqQ}Xlq>7&GGh^fS77Ga6iqDC;EvmvS?M){+ByvwGVY~C%us`sU}F3r0fPZ`aZ1nKh;&244AuhKfZiYkQfbroq|}yXFp~AxVd2B z+nP|EN?+D6hVw3I?U?`m=;_+CYoA(|o+#tuI`zPO_WDj*wl%bZsDM2aU-GDZ{m6_^v$&svjWO!LwT4Dj#Oe*8b0Jgm!)Z+Y9{tz-$J6WL8+)A-gESQfeG^2d@W;4p#%0vohePCshN3Jugu{P-KokXbCFY zJksH7^w;vW>?tjTS05=CIa7;C084@r|1mX~mT){!rB4cu!Xn-)ylhBkqaOh&L3!Cv z96*5%1uGLE)icD@iwCU=Xm{ngO12z#Z<)@i*;UQSW4RRP&z(I`$*KDoKC0 zSS->oy%Br{M152CS!!cc5T9bTrZ{v}FcR$ShI})B6beowe}T(4^GEK-mIOXSi)D6sRou-#v%({<*PdnW~31uC-Nz^ zN{IG-24L1%Bh7kn-V;{^Q&+@^RbghRxZ-6jp{k(`|^`zsf z3jKwPM!f}nVepUrl9$B6s)hn@cEaW{?zuI5FxKa$-M32P-Pf;+pGJ2`Pnhd905cGvI*yZFn~6~XS?}EM$yaHw#*#rDRGTcyjR>%YjfE0Xpk+xKyKg4 z$xDZ<8~ZLLPDC~ zYLg3IHqy}#8&Q*)ESH!;ssxD~`H)Ma3M1*9)-(SsOE(DNNGjQ6THNE8FI2mpo?W-N ztdXJIgVGicK_4q9;fSVi|E0QYpX-76m3vTLoqEWIwppro4N4?x-coi$_ndffk$H1t z>2_PCtuwk1ysPrnXBZFXKs3UvwGlJEU>le;82H^Z#q`|6Eg`}pzx%=mqMm2g$nukpo*u18_*K(=eCmYrh! zpFQsDp$AGJRePC73g_WlT99S;rZzw`#e!?>dp4#TIou>qAd=0{!&)3A=#YcGK=a?PJ9w*xD;ut^sFuKHv1ca2kD7N)e8N@F<|(9b-O zoFOMsha$itQfz9k&Au#z&nt`VCfMW~y=44Lt?%UCMEp!*2WR`9C8+7TtVpC@(P~KqA4(A`acQ$7+4HM$;?ow8hseipTdM=@ zx;+gUkqM5}?Ah_$oB(@G+`gJo0;gNVKTF_oQMfLs>kOU>?^^3yW~ikKg*9G_@G{On zsMwF5i7}Gm2Mb5SV2tqdkTUu?2Oj)Qn-Gy0l3`2 zZi-ECVfg@6)-mJm^h2Wy8w}7fAiR26R`%gkJ^Y$+&U(g}B`l4r9~>6BH^g$c6d?Y8 z0?iXL>^{Q`gdQ3i@Vh@tk!w$Jq(|HmM7qkQ8_Zg~hl=!ZWYA3BVD+19Yo#d>mB4J( z9FJ2Wip#`rr9`bmu(K|A(w)?%;h730h>^IdG$a`&Hm_X_aB8NyF_;!*)~pgLn$Tf5 z7uEExdu|fYO(O{z2?q3nl2+1@fszW&%BZECj}%20B%Xj$rQjl=TPyuu~XDknhJ1ayT%qr zXvhXoWbN*49zfK_PRl3sYViA=E*|Rszw7=FW2@WdqTjdsDfF-mhtuWOyy!cV1)L3! zTEQq}wyUAsT8Jc+Q@A&`FQ)a3gVP~dE(OWyL52Xko?M0yfbr+~hg()!leT~np$wwd zJN*STXh;*g<}6T1!$Jm?YE)LtAI5T;Ka|c1x9(MB2hKLv_*bjdwy<($N4#S~ox|g~ zc!yq5w;!#p4SM!ko?VO#Dm}1osb9!-^$qF^L0M)qTq|)STWd@L{{Un(psR6F)|yQP z?5u1oG~D>slOG^p;>4*Q9cam*wp?sntEABf2TQQlmesA4H6Sy+%l*IYnVjDh+6TRW z8(-;MIexPhw)TOeE1mFu``hF%?fdZ!Lni+Kg?QbcsLNez?nXk{6_rZ3D&uuwV_s^b ztA?SLss@5~hD+blO;yrLj`4Ftt%<2y5SBC2Fe3HcTMW?{n&j2-u}(;$_0ocIBfSQK zk=Be{*DZ#E4Pi+pD|-=20^AHt%6zty@*CS{{YUrTQmVjn2ntf zjX2%P@@)sG5&USqjDpBTUvrqZ5K97VYj=Rm#Q5id2Yw%< z*Hp@qeuLO#BkDG!Xu-CJD;=SkL!*u?q+AhGdKkoa?!%YanB?LHm$w~RPOTT$#eD0@ z?R2BvO}xC1Pp#XXEx5Hak-)j%Ppg}>e>NaIO-*tn8O~&dcl)_u; z(z<>S*3gaU$RL-z!~yxab)zN#?j4V{=SOLvQh!s614)w&FZ(3SjMsGk0B$36X*+b* zVYE$Nc3-Na){l$-0I>DQ*5+aRH*~;|u2d|R{XkuR8uGIBa%26F?#YN)k-c-Z1D%Z9 z?_C&}rJ+lGE3%;`00N6c5gL(|GQxYSia zJqt3cqH-RAz=3O<9sCc4C`#m6ZN}(Ti5B7o0|H8m#?r6&$pILX zuh!O}6S&&QW3W)SvyeCF2DhLPye{C~x4XlWsfJHRLXCYachkVtaNEPoOD_}ABMNpp z9f|^EB3QZR_GP*sRJcRF%O9A7ZAbe*z-N$z>?3ac)A# z^7+z)A1B@YMo7sAsZ$!t>DxeMrq#J$cpcX4x#B4>I)GgACfRTM}8pP?!iHk{!Sd0@Y$^F5H zO3zas7bIMk)~}bnBUPY_2LLtdZ&+hvPlBvA`sVd-=>QCR2bIn_R4FcCT-()TS-9x0%mtHZn6j zgpOvhafE1$%LG*DE5#{ME=?kYF4aF;A1yL(1s6M-! zmC$eCXao?UH5wkALed$Q* z#;*j^JA)CEx?-Rgyrb|Mnx|f~bS78rJ#*l(vF?rst2ECS{GtRr%@6XeZc7e`{ z@ezOgFaH2Km9|900i#A9o(jNR?0j!Xj7Tst4j+_iLDU=3F@FdOhB^!AwWQE-OCVrZ z(sj5$%CHGp+B={M=Hp)~!7q%9UC2CztSbbAo?}8lzo~kCXn>eTPzwv|bv{*%1?o|d zvZ+2pYt}K&Kl_eJ*FtE>An}3)tzHZr1bBidL5LgNC;-agPQLEt`cH-R-m_+c4=3d+G#yFW$6@C{W3v0U1~~}lVz&UBoo)9tp*r_5 zsB49hh&C4rZF`f@QiQ5DMCI)cQgX8$_3+lQfU}Ui5;I)L0k|F@dWAY*8pP4G5U{Y1 zirA5@xQmVVsSI=7h|3zSiBfo~-q+ga&asZNK_iV1r*a6l-hfKv5VETBV2*5c1Ow)4 z1RUt$Ls?1x03(ZfPw88a9~w}SS$P0K^Z{b4x`1u3-hqbS6U{JpLCpUEk1PCcr=tzF z(@MTKDWNR2H-RkF3|xl7_oXp>X4?KVti{5{l)#n0^d6w;ZgdS zT?z7Gaa^{;YVM1ywYJPpkB-wBrMIVBQ&kMvm5psYs^6uJ$KI&~(;5705KG-r*l7Z^ zhwq@So}GWPe2Hg%)7Cp@a_pZQ!N<++u5s5gPhu7dSeGk*Rkg4i_*bKr-UkFoKhB>Q zD;onl8dRGd%czwN!Q?s-^r^jtgEJo{w=G#kl~0G4+L0Kwc>6@J*d2frOj{bFs-uC# z>PXZ8eCq_VIN~PPYaKP#)UXn`3Ouah>;U2iffT}pD2+5bAIiQJf^aEUH_=Ui2VH9z zn@Ws^G7mkg80=w?SROju{S9Li2_>)kM_&QsT1$?7tT6eUPk%*;&>cvKW3+dPX);=e>IPIH;`A0#sVutZ(}g0Ij&Gx2sJ_(1ZLf zK)~{(at0DDl#X{FmzUa#W)n1>_p?l2xBmd|@f?;oa-YgSSGM-h`5K+7)~ICs!#j7$ zdq;7|_LMd^a3|G)AmZcce@p$11$odeqd^je9^mB1PD(651Ffx2Df2Qrv?!wSpYjoV zgKd|@&~i)Vuq*!nb3R15-zNR0BY*bp z%0F$1V{p!={L+8a+I^~*l6cf1_iniO^bvt&pCHQ`4-Pt!_cYfhZApMx6kg&s0JmSg zO%pqIMPR5&VRc^}bq1Kkv89chi~1bsVr?}P6ATB!3ab&sh5(h?#)*L9TNxuK)Et|R zm#hGo7t~d?wzoZN1iZ9=orS>YIv$h^R~xAe->YyotP;oK#^du7si0y~GqRO$N{b7CFSrI~*aXc&}dn6W33^uO5carK}Q7IKPI zhbgwC*c}ax0~Ry6USWD!bQ%la=~yL^qNI%}^pm3tk}Ng1)YdRJ6?5(0=as^ZF2CAO z-m!$8+p^$iEBlaLfzi`Kgm^W@i1MK5tOx%pB_mLM{iI2IWqvW0yQar*Yz~K zTJal{j{VqlbwYr8O@)Z-r}U#HKK85&Gm#sBI4-u=rMx$-1|yOh<3846gaagksDL028XyW$fyKmwu)Yh@&Bhi~=I^5E;6GvTH zw2yn8zQ0Ng#$UeowS!*cz>h$()tvwV^Af4qJ}ZK;z6(%2k@QeKF4BWngXy zAZRzfl@7&rXHQ-3j8}8SITNH~c^D%7NvJBzNa?eMCZ85qJ4bKJ?hyK6?YQq89&n;1y%g;GPM4cWFgyi1AhQ@irFI>0I{E z%$VmNPD^{6Z?3x1qHM9aup&E-g8Lu6Fb>Iv^EI4$m}v#qGpnlDhL8|Nx4wk`08v=QhA`x{{Sqk;$DlSK z5!0;$A3cFBxBw70x{vtQ2@xRV1DI%)!ZZYZ^bT{zj_oNuC!l0J`s%>y@X1;>nRg7E+3bDpO zx#TUVw^L*H6^u|B8Hg9;!pfxE`S@*E#B@};N33^Z!=Ui{QiS^&Qw+;2WN{v}0PIxw z{dK0Lp~xDPdQWc7k2?-5h5W1sA^xtT@v7di#X@h8_otoC1~I%NOqmqu$NL4mI?|*v z(4!X`GOOtYbQYn;2jnor#D|lk17O4rPSjc!L&gp$ADEB&jVuC6<;LxbF5{bY1d~jY zG?^cnW+phs5hz7|V^1qoyA_0vp0m}9dq-@RQoX|{6pk-kXm?|63G%9Sdd^j?`~$I_ zKWZBH%ovBoVC}2_0CHQ0N-0Lm2qa=sL_aDPUM{k{zjyUvT4M`O@Ii(*V9wI88L1+YcmGWpGagXq;y5#0TCh)?+^Y{tY{Pm1IPB$E ha{mAyLE+^ -
-
+
+
@@ -14,9 +14,20 @@

{{ _('Board Member') }}

- +
-
+
+ + + + +

+ {{ _('Caryn Marooney') }} +

+ + +
+
@@ -27,7 +38,27 @@

{{ _('Board Member') }}

- + + +
+
+ + + + +

+ {{ _('Kevin Wang') }} +

+
+ +
+ + + + + + +
diff --git a/website/templates/index/hero.html b/website/templates/index/hero.html index d08b1f8d264..865e3d4b74e 100644 --- a/website/templates/index/hero.html +++ b/website/templates/index/hero.html @@ -3,7 +3,7 @@

- ClickHouse, Inc Has Arrived + ClickHouse v21.10 Released

@@ -11,12 +11,12 @@

- Learn More - Online Demo + Learn More + Release Webinar

- Read the Blog Post + Read the Blog Post

@@ -28,10 +28,13 @@
-

ClickHouse v21.10 Release Webinar

+

ClickHouse Announces $250 Million in Funding

+ +

Raising the Company’s Valuation to $2B

- Watch on YouTube + Read the News + Read the Blog Post
From 4d857edab1557167778bd7e110f80b15e65c1922 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 28 Oct 2021 17:11:00 +0300 Subject: [PATCH 169/173] [website] update press links (#30808) * [website] update press links * Update hero.html --- website/templates/company/press.html | 27 +++++++++++++++++++++++++++ website/templates/index/hero.html | 3 ++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/website/templates/company/press.html b/website/templates/company/press.html index 8265b68b063..fea088eec3f 100644 --- a/website/templates/company/press.html +++ b/website/templates/company/press.html @@ -11,6 +11,33 @@
+
+

+ {{ _('10/28/21') }} +

+

+ {{ _('ClickHouse Raises $250M Series B To Scale Groundbreaking OLAP Database Management System Globally') }} +

+

+ {{ _('The investment was led by Coatue and Altimeter, with participation from Index Ventures, Benchmark, Lightspeed, Redpoint, Almaz, Yandex N.V., FirstMark and Lead Edge.') }} +

+ {{ _('Read More') }} +
+ +
+ +

+ {{ _('10/28/21') }} +

+

+ {{ _('Bloomberg Exclusive') }} +

+

+ {{ _('ClickHouse Valued at $2 Billion in Round After Yandex Spinout.') }} +

+ {{ _('Read More') }} +
+

diff --git a/website/templates/index/hero.html b/website/templates/index/hero.html index 865e3d4b74e..35799d3b8d7 100644 --- a/website/templates/index/hero.html +++ b/website/templates/index/hero.html @@ -33,8 +33,9 @@

Raising the Company’s Valuation to $2B

+ Read the Blog Post Read the News - Read the Blog Post + Read the Press Release
From 7bfef3038444083e7fe644f8538f9175736a4e97 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 28 Oct 2021 18:09:03 +0300 Subject: [PATCH 170/173] Update overview.html (#30810) --- website/templates/company/overview.html | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/website/templates/company/overview.html b/website/templates/company/overview.html index e5759227c86..2d95a57ddd8 100644 --- a/website/templates/company/overview.html +++ b/website/templates/company/overview.html @@ -1,14 +1,16 @@
- +

- Creators of the online analytical processing (OLAP) database management system ClickHouse have announced their decision to officially incorporate as a company. The creator of ClickHouse, Alexey Milovidov (CTO), will be joined by co-founders and seasoned enterprise software executives, Yury Izrailevsky (President, Product and Engineering) and Aaron Katz (CEO), along with nearly $50M in Series A funding led by Index Ventures and Benchmark. + ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time.

- -

- ClickHouse is an open-source, column-oriented OLAP database management system that allows users to generate analytical reports using SQL queries in real-time. Its technology works 100-1,000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance. +

+ Its technology works 100-1000x faster than traditional database management systems and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.

- +

+ In September of 2021 in San Fransisco, CA, ClickHouse incorporated to house the open source technology with an initial $50 million investment from Index Ventures and Benchmark Capital. On October 28, 2021 the company received Series B funding totaling $250 million at an evaluation of $2 billion from Coatue Management, Altimeter Capital, and other investors. The company continues to build the open source project and engineering the cloud technology. +

+
From ba3c4c9958d7ef3def38b6c8531826d80e419bc6 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 28 Oct 2021 18:22:43 +0300 Subject: [PATCH 171/173] Update investors.html (#30811) --- website/templates/company/investors.html | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/website/templates/company/investors.html b/website/templates/company/investors.html index 43b7e86a5dc..cf7b3631060 100644 --- a/website/templates/company/investors.html +++ b/website/templates/company/investors.html @@ -3,8 +3,21 @@
-
+
+ + + +

+ {{ _('Mike Volpi') }} +

+

+ {{ _('Board Member') }} +

+ + +
+
@@ -15,8 +28,9 @@ {{ _('Board Member') }}

+
-
+
@@ -26,20 +40,6 @@ -
-
- - - - -

- {{ _('Mike Volpi') }} -

-

- {{ _('Board Member') }} -

- -
@@ -55,10 +55,10 @@
- - - + + +
From a4b1ce58e91935bca9510576837893a21d56fc2c Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 28 Oct 2021 18:26:17 +0300 Subject: [PATCH 172/173] [website] add Lightspeed post link (#30812) --- website/templates/company/press.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/website/templates/company/press.html b/website/templates/company/press.html index fea088eec3f..ddb2cf584ac 100644 --- a/website/templates/company/press.html +++ b/website/templates/company/press.html @@ -38,6 +38,20 @@ {{ _('Read More') }}
+
+ +

+ {{ _('10/28/21') }} +

+

+ {{ _('Lightspeed Perspective') }} +

+

+ {{ _('Why Lightspeed invested in ClickHouse: a database built for speed.') }} +

+ {{ _('Read More') }} +
+

From 796fe54ed7c3af19a33cb227605e36a48942ba4c Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 28 Oct 2021 18:28:59 +0300 Subject: [PATCH 173/173] [blog] update clickhouse-raises-250m-series-b.md (#30813) * Update clickhouse-raises-250m-series-b.md * Update clickhouse-raises-250m-series-b.md * Update clickhouse-raises-250m-series-b.md --- website/blog/en/2021/clickhouse-raises-250m-series-b.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/blog/en/2021/clickhouse-raises-250m-series-b.md b/website/blog/en/2021/clickhouse-raises-250m-series-b.md index 5933782c924..a443ad61cb7 100644 --- a/website/blog/en/2021/clickhouse-raises-250m-series-b.md +++ b/website/blog/en/2021/clickhouse-raises-250m-series-b.md @@ -3,13 +3,13 @@ title: 'ClickHouse raises a $250M Series B at a $2B valuation...and we are hirin image: 'https://blog-images.clickhouse.com/en/2021/clickhouse-raises-250m-series-b/featured.jpg' date: '2021-10-28' author: '[Dorota Szeremeta](https://www.linkedin.com/in/dorota-szeremeta-a849b7/)' -tags: ['company', 'community'] +tags: ['company', 'investment'] --- -Today, we are excited to announce that ClickHouse has raised $250 million in Series B funding at a $2B valuation. The round was led by Coatue and Altimeter, with participation from existing investors Index, Benchmark, Yandex, Almaz, and Lead Edge alongside new investors Lightspeed, Redpoint, and FirstMark. We are thrilled to partner with such an accomplished group. Not only is it a testament to the disruptive potential of our technology and our momentum to date, but more importantly, it’s an aggressive investment in our future. This funding will enable ClickHouse to grow our team, scale the product, and continue building a world-class, commercial-grade cloud solution that’s secure, compliant, and convenient for any customer to use. (You can read more about the funding round in our press release here). +Today, we are excited to announce that ClickHouse has raised $250 million in Series B funding at a $2B valuation. The round was led by Coatue and Altimeter, with participation from existing investors Index, Benchmark, Yandex, Almaz, and Lead Edge alongside new investors Lightspeed, Redpoint, and FirstMark. We are thrilled to partner with such an accomplished group. Not only is it a testament to the disruptive potential of our technology and our momentum to date, but more importantly, it’s an aggressive investment in our future. This funding will enable ClickHouse to grow our team, scale the product, and continue building a world-class, commercial-grade cloud solution that’s secure, compliant, and convenient for any customer to use. (You can read more about the funding round in our press release [here](https://www.businesswire.com/news/home/20211028005287/en)). For the past five years, ClickHouse thrived as a popular open-source product— with a dedicated user community. ClickHouse thrived for a simple reason—we deliver high query processing speed and data storage efficiency that is unmatched. ClickHouse is capable of processing 100+ PBs of data with more than 100 billion records inserted every day. It is 100-1000X faster than traditional approaches and this speed and efficiency has attracted customers that are household names. The commercial potential for the product was clear - and in September 2021, we decided to combine resources and officially incorporate as a company. We took the team of core ClickHouse developers led by ClickHouse creator [Alexey Milovidov](https://www.linkedin.com/in/alexey-milovidov-7b658135/) and brought in a world-class business team led by [Aaron Katz](https://www.linkedin.com/in/aaron-katz-5762094/) and a cloud engineering team led by [Yury Izrailevsky](https://www.linkedin.com/in/yuryizrailevsky/). Fueled by this large investment, we are hiring. We plan to double our team this year, and double it again next year. We are calling on all visionaries, builders, and contributors to join us as we build ClickHouse into a transformative, paradigm-shifting company. Our mission is to become the first choice of analytical database management systems. We are relentlessly focused on building an enterprise-grade, highly secure, and fault tolerant, multi-tenant service in the cloud that is accessible to organizations across sizes and sectors. -If this sounds like something you want to be a part of, now’s the time to join us. To learn more about the positions we are hiring for, and what we are looking for in new hires, visit [www.clickhouse.com/careers](www.clickhouse.com/careers) +If this sounds like something you want to be a part of, now’s the time to join us. To learn more about the positions we are hiring for, and what we are looking for in new hires, visit [www.clickhouse.com/careers](/careers/)