diff --git a/README.md b/README.md index abaf27abf11..3270cd19671 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,6 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else Upcoming meetups -* [Barcelona Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096876/) - November 12 * [Ghent Meetup](https://www.meetup.com/clickhouse-belgium-user-group/events/303049405/) - November 19 * [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21 * [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26 @@ -53,6 +52,7 @@ Upcoming meetups Recently completed meetups +* [Barcelona Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096876/) - November 12 * [Madrid Meetup](https://www.meetup.com/clickhouse-spain-user-group/events/303096564/) - October 22 * [Singapore Meetup](https://www.meetup.com/clickhouse-singapore-meetup-group/events/303212064/) - October 3 * [Jakarta Meetup](https://www.meetup.com/clickhouse-indonesia-user-group/events/303191359/) - October 1 diff --git a/base/base/defines.h b/base/base/defines.h index 5685a6d9833..a0c3c0d1de5 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -145,6 +145,7 @@ #define TSA_TRY_ACQUIRE_SHARED(...) __attribute__((try_acquire_shared_capability(__VA_ARGS__))) /// function tries to acquire a shared capability and returns a boolean value indicating success or failure #define TSA_RELEASE_SHARED(...) __attribute__((release_shared_capability(__VA_ARGS__))) /// function releases the given shared capability #define TSA_SCOPED_LOCKABLE __attribute__((scoped_lockable)) /// object of a class has scoped lockable capability +#define TSA_RETURN_CAPABILITY(...) __attribute__((lock_returned(__VA_ARGS__))) /// to return capabilities in functions /// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function) /// They use a lambda function to apply function attribute to a single statement. This enable us to suppress warnings locally instead of diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 0fe9a409ee4..e6bde845c4e 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -113,7 +113,9 @@ RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \ /var/lib/apt/lists/* \ /var/cache/debconf \ /tmp/* \ - && apt-get autoremove --purge -yq dirmngr gnupg2 + && apt-get autoremove --purge -yq dirmngr gnupg2 \ + && chmod ugo+Xrw -R /etc/clickhouse-server /etc/clickhouse-client +# The last chmod is here to make the next one is No-op in docker official library Dockerfile # post install # we need to allow "others" access to clickhouse folder, because docker container diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index c5f92ccdf68..ca4938b1a47 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -597,6 +597,30 @@ If number of tables is greater than this value, server will throw an exception. 400 ``` +## max\_replicated\_table\_num\_to\_throw {#max-replicated-table-num-to-throw} +If number of replicated tables is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine. + +**Example** +```xml +400 +``` + +## max\_dictionary\_num\_to\_throw {#max-dictionary-num-to-throw} +If number of dictionaries is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine. + +**Example** +```xml +400 +``` + +## max\_view\_num\_to\_throw {#max-view-num-to-throw} +If number of views is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine. + +**Example** +```xml +400 +``` + ## max\_database\_num\_to\_throw {#max-table-num-to-throw} If number of _database is greater than this value, server will throw an exception. 0 means no limitation. Default value: 0 diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 2357b5b2fdd..34dc6e996ee 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -4773,7 +4773,7 @@ Result: ## toUTCTimestamp -Convert DateTime/DateTime64 type value from other time zone to UTC timezone timestamp +Convert DateTime/DateTime64 type value from other time zone to UTC timezone timestamp. This function is mainly included for compatibility with Apache Spark and similar frameworks. **Syntax** @@ -4799,14 +4799,14 @@ SELECT toUTCTimestamp(toDateTime('2023-03-16'), 'Asia/Shanghai'); Result: ``` text -┌─toUTCTimestamp(toDateTime('2023-03-16'),'Asia/Shanghai')┐ +┌─toUTCTimestamp(toDateTime('2023-03-16'), 'Asia/Shanghai')┐ │ 2023-03-15 16:00:00 │ └─────────────────────────────────────────────────────────┘ ``` ## fromUTCTimestamp -Convert DateTime/DateTime64 type value from UTC timezone to other time zone timestamp +Convert DateTime/DateTime64 type value from UTC timezone to other time zone timestamp. This function is mainly included for compatibility with Apache Spark and similar frameworks. **Syntax** @@ -4832,7 +4832,7 @@ SELECT fromUTCTimestamp(toDateTime64('2023-03-16 10:00:00', 3), 'Asia/Shanghai') Result: ``` text -┌─fromUTCTimestamp(toDateTime64('2023-03-16 10:00:00',3),'Asia/Shanghai')─┐ +┌─fromUTCTimestamp(toDateTime64('2023-03-16 10:00:00',3), 'Asia/Shanghai')─┐ │ 2023-03-16 18:00:00.000 │ └─────────────────────────────────────────────────────────────────────────┘ ``` diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index bf67db8ff2e..e73f61dde83 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -12,9 +12,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -43,29 +46,24 @@ namespace CurrentMetrics namespace { -/// Outputs sizes of uncompressed and compressed blocks for compressed file. +/// Outputs method, sizes of uncompressed and compressed blocks for compressed file. void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out) { while (!in.eof()) { - in.ignore(16); /// checksum - - char header[COMPRESSED_BLOCK_HEADER_SIZE]; - in.readStrict(header, COMPRESSED_BLOCK_HEADER_SIZE); - - UInt32 size_compressed = unalignedLoad(&header[1]); + UInt32 size_compressed; + UInt32 size_decompressed; + auto codec = DB::getCompressionCodecForFile(in, size_compressed, size_decompressed, true /* skip_to_next_block */); if (size_compressed > DBMS_MAX_COMPRESSED_SIZE) throw DB::Exception(DB::ErrorCodes::TOO_LARGE_SIZE_COMPRESSED, "Too large size_compressed. Most likely corrupted data."); - UInt32 size_decompressed = unalignedLoad(&header[5]); - + DB::writeText(queryToString(codec->getFullCodecDesc()), out); + DB::writeChar('\t', out); DB::writeText(size_decompressed, out); DB::writeChar('\t', out); DB::writeText(size_compressed, out); DB::writeChar('\n', out); - - in.ignore(size_compressed - COMPRESSED_BLOCK_HEADER_SIZE); } } diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index 1ad7e7996c7..27b46a112e2 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -88,6 +88,7 @@ void FunctionNode::resolveAsFunction(FunctionBasePtr function_value) function_name = function_value->getName(); function = std::move(function_value); kind = FunctionKind::ORDINARY; + nulls_action = NullsAction::EMPTY; } void FunctionNode::resolveAsAggregateFunction(AggregateFunctionPtr aggregate_function_value) @@ -95,6 +96,12 @@ void FunctionNode::resolveAsAggregateFunction(AggregateFunctionPtr aggregate_fun function_name = aggregate_function_value->getName(); function = std::move(aggregate_function_value); kind = FunctionKind::AGGREGATE; + /** When the function is resolved, we do not need the nulls action anymore. + * The only thing that the nulls action does is map from one function to another. + * Thus, the nulls action is encoded in the function name and does not make sense anymore. + * Keeping the nulls action may lead to incorrect comparison of functions, e.g., count() and count() IGNORE NULLS are the same function. + */ + nulls_action = NullsAction::EMPTY; } void FunctionNode::resolveAsWindowFunction(AggregateFunctionPtr window_function_value) diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index bf99c014826..722c1e19b7e 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -48,9 +48,15 @@ ASTPtr JoinNode::toASTTableJoin() const auto join_expression_ast = children[join_expression_child_index]->toAST(); if (is_using_join_expression) - join_ast->using_expression_list = std::move(join_expression_ast); + { + join_ast->using_expression_list = join_expression_ast; + join_ast->children.push_back(join_ast->using_expression_list); + } else - join_ast->on_expression = std::move(join_expression_ast); + { + join_ast->on_expression = join_expression_ast; + join_ast->children.push_back(join_ast->on_expression); + } } return join_ast; diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp index 17a765a068b..cd2577c3d76 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.cpp +++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp @@ -85,10 +85,9 @@ QueryTreeNodePtr createResolvedFunction(const ContextPtr & context, const String } FunctionNodePtr createResolvedAggregateFunction( - const String & name, const QueryTreeNodePtr & argument, const Array & parameters = {}, NullsAction action = NullsAction::EMPTY) + const String & name, const QueryTreeNodePtr & argument, const Array & parameters = {}) { auto function_node = std::make_shared(name); - function_node->setNullsAction(action); if (!parameters.empty()) { @@ -100,7 +99,7 @@ FunctionNodePtr createResolvedAggregateFunction( function_node->getArguments().getNodes() = { argument }; AggregateFunctionProperties properties; - auto aggregate_function = AggregateFunctionFactory::instance().get(name, action, {argument->getResultType()}, parameters, properties); + auto aggregate_function = AggregateFunctionFactory::instance().get(name, NullsAction::EMPTY, {argument->getResultType()}, parameters, properties); function_node->resolveAsAggregateFunction(std::move(aggregate_function)); return function_node; diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 4443f83596f..0005aa6c8bc 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -3,7 +3,6 @@ #include #include -#include "Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.h" #include #include @@ -16,39 +15,39 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include - +#include +#include +#include +#include namespace DB { diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 390418494e7..03ebd893c47 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -676,6 +677,8 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden "tuple"}); } } + + logProcessorProfile(context, io.pipeline.getProcessors()); } scalars_cache.emplace(node_with_hash, scalar_block); diff --git a/src/Backups/BackupConcurrencyCheck.cpp b/src/Backups/BackupConcurrencyCheck.cpp index 8b29ae41b53..a67d241845d 100644 --- a/src/Backups/BackupConcurrencyCheck.cpp +++ b/src/Backups/BackupConcurrencyCheck.cpp @@ -14,12 +14,12 @@ namespace ErrorCodes BackupConcurrencyCheck::BackupConcurrencyCheck( - const UUID & backup_or_restore_uuid_, bool is_restore_, bool on_cluster_, + const String & zookeeper_path_, bool allow_concurrency_, BackupConcurrencyCounters & counters_) - : is_restore(is_restore_), backup_or_restore_uuid(backup_or_restore_uuid_), on_cluster(on_cluster_), counters(counters_) + : is_restore(is_restore_), on_cluster(on_cluster_), zookeeper_path(zookeeper_path_), counters(counters_) { std::lock_guard lock{counters.mutex}; @@ -32,7 +32,7 @@ BackupConcurrencyCheck::BackupConcurrencyCheck( size_t num_on_cluster_restores = counters.on_cluster_restores.size(); if (on_cluster) { - if (!counters.on_cluster_restores.contains(backup_or_restore_uuid)) + if (!counters.on_cluster_restores.contains(zookeeper_path)) ++num_on_cluster_restores; } else @@ -47,7 +47,7 @@ BackupConcurrencyCheck::BackupConcurrencyCheck( size_t num_on_cluster_backups = counters.on_cluster_backups.size(); if (on_cluster) { - if (!counters.on_cluster_backups.contains(backup_or_restore_uuid)) + if (!counters.on_cluster_backups.contains(zookeeper_path)) ++num_on_cluster_backups; } else @@ -64,9 +64,9 @@ BackupConcurrencyCheck::BackupConcurrencyCheck( if (on_cluster) { if (is_restore) - ++counters.on_cluster_restores[backup_or_restore_uuid]; + ++counters.on_cluster_restores[zookeeper_path]; else - ++counters.on_cluster_backups[backup_or_restore_uuid]; + ++counters.on_cluster_backups[zookeeper_path]; } else { @@ -86,7 +86,7 @@ BackupConcurrencyCheck::~BackupConcurrencyCheck() { if (is_restore) { - auto it = counters.on_cluster_restores.find(backup_or_restore_uuid); + auto it = counters.on_cluster_restores.find(zookeeper_path); if (it != counters.on_cluster_restores.end()) { if (!--it->second) @@ -95,7 +95,7 @@ BackupConcurrencyCheck::~BackupConcurrencyCheck() } else { - auto it = counters.on_cluster_backups.find(backup_or_restore_uuid); + auto it = counters.on_cluster_backups.find(zookeeper_path); if (it != counters.on_cluster_backups.end()) { if (!--it->second) diff --git a/src/Backups/BackupConcurrencyCheck.h b/src/Backups/BackupConcurrencyCheck.h index 048a23a716a..a1baeff5464 100644 --- a/src/Backups/BackupConcurrencyCheck.h +++ b/src/Backups/BackupConcurrencyCheck.h @@ -1,7 +1,8 @@ #pragma once -#include +#include #include +#include #include #include @@ -19,9 +20,9 @@ public: /// Checks concurrency of a BACKUP operation or a RESTORE operation. /// Keep a constructed instance of BackupConcurrencyCheck until the operation is done. BackupConcurrencyCheck( - const UUID & backup_or_restore_uuid_, bool is_restore_, bool on_cluster_, + const String & zookeeper_path_, bool allow_concurrency_, BackupConcurrencyCounters & counters_); @@ -31,8 +32,8 @@ public: private: const bool is_restore; - const UUID backup_or_restore_uuid; const bool on_cluster; + const String zookeeper_path; BackupConcurrencyCounters & counters; }; @@ -47,8 +48,8 @@ private: friend class BackupConcurrencyCheck; size_t local_backups TSA_GUARDED_BY(mutex) = 0; size_t local_restores TSA_GUARDED_BY(mutex) = 0; - std::unordered_map on_cluster_backups TSA_GUARDED_BY(mutex); - std::unordered_map on_cluster_restores TSA_GUARDED_BY(mutex); + std::unordered_map on_cluster_backups TSA_GUARDED_BY(mutex); + std::unordered_map on_cluster_restores TSA_GUARDED_BY(mutex); std::mutex mutex; }; diff --git a/src/Backups/BackupCoordinationCleaner.cpp b/src/Backups/BackupCoordinationCleaner.cpp index 1f5068a94de..47095f27eb3 100644 --- a/src/Backups/BackupCoordinationCleaner.cpp +++ b/src/Backups/BackupCoordinationCleaner.cpp @@ -4,31 +4,29 @@ namespace DB { -BackupCoordinationCleaner::BackupCoordinationCleaner(const String & zookeeper_path_, const WithRetries & with_retries_, LoggerPtr log_) - : zookeeper_path(zookeeper_path_), with_retries(with_retries_), log(log_) +BackupCoordinationCleaner::BackupCoordinationCleaner(bool is_restore_, const String & zookeeper_path_, const WithRetries & with_retries_, LoggerPtr log_) + : is_restore(is_restore_), zookeeper_path(zookeeper_path_), with_retries(with_retries_), log(log_) { } -void BackupCoordinationCleaner::cleanup() +bool BackupCoordinationCleaner::cleanup(bool throw_if_error) { - tryRemoveAllNodes(/* throw_if_error = */ true, /* retries_kind = */ WithRetries::kNormal); + WithRetries::Kind retries_kind = throw_if_error ? WithRetries::kNormal : WithRetries::kErrorHandling; + return cleanupImpl(throw_if_error, retries_kind); } -bool BackupCoordinationCleaner::tryCleanupAfterError() noexcept -{ - return tryRemoveAllNodes(/* throw_if_error = */ false, /* retries_kind = */ WithRetries::kNormal); -} - -bool BackupCoordinationCleaner::tryRemoveAllNodes(bool throw_if_error, WithRetries::Kind retries_kind) +bool BackupCoordinationCleaner::cleanupImpl(bool throw_if_error, WithRetries::Kind retries_kind) { { std::lock_guard lock{mutex}; - if (cleanup_result.succeeded) - return true; - if (cleanup_result.exception) + if (succeeded) { - if (throw_if_error) - std::rethrow_exception(cleanup_result.exception); + LOG_TRACE(log, "Nodes from ZooKeeper are already removed"); + return true; + } + if (tried) + { + LOG_INFO(log, "Skipped removing nodes from ZooKeeper because because earlier we failed to do that"); return false; } } @@ -44,16 +42,18 @@ bool BackupCoordinationCleaner::tryRemoveAllNodes(bool throw_if_error, WithRetri }); std::lock_guard lock{mutex}; - cleanup_result.succeeded = true; + tried = true; + succeeded = true; return true; } catch (...) { - LOG_TRACE(log, "Caught exception while removing nodes from ZooKeeper for this restore: {}", + LOG_TRACE(log, "Caught exception while removing nodes from ZooKeeper for this {}: {}", + is_restore ? "restore" : "backup", getCurrentExceptionMessage(/* with_stacktrace= */ false, /* check_embedded_stacktrace= */ true)); std::lock_guard lock{mutex}; - cleanup_result.exception = std::current_exception(); + tried = true; if (throw_if_error) throw; diff --git a/src/Backups/BackupCoordinationCleaner.h b/src/Backups/BackupCoordinationCleaner.h index 43e095d9f33..c760a3611f9 100644 --- a/src/Backups/BackupCoordinationCleaner.h +++ b/src/Backups/BackupCoordinationCleaner.h @@ -12,14 +12,14 @@ namespace DB class BackupCoordinationCleaner { public: - BackupCoordinationCleaner(const String & zookeeper_path_, const WithRetries & with_retries_, LoggerPtr log_); + BackupCoordinationCleaner(bool is_restore_, const String & zookeeper_path_, const WithRetries & with_retries_, LoggerPtr log_); - void cleanup(); - bool tryCleanupAfterError() noexcept; + bool cleanup(bool throw_if_error); private: - bool tryRemoveAllNodes(bool throw_if_error, WithRetries::Kind retries_kind); + bool cleanupImpl(bool throw_if_error, WithRetries::Kind retries_kind); + const bool is_restore; const String zookeeper_path; /// A reference to a field of the parent object which is either BackupCoordinationOnCluster or RestoreCoordinationOnCluster. @@ -27,13 +27,8 @@ private: const LoggerPtr log; - struct CleanupResult - { - bool succeeded = false; - std::exception_ptr exception; - }; - CleanupResult cleanup_result TSA_GUARDED_BY(mutex); - + bool tried TSA_GUARDED_BY(mutex) = false; + bool succeeded TSA_GUARDED_BY(mutex) = false; std::mutex mutex; }; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 8bd6b4d327d..402e789eacb 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -11,12 +11,11 @@ namespace DB { BackupCoordinationLocal::BackupCoordinationLocal( - const UUID & backup_uuid_, bool is_plain_backup_, bool allow_concurrent_backup_, BackupConcurrencyCounters & concurrency_counters_) : log(getLogger("BackupCoordinationLocal")) - , concurrency_check(backup_uuid_, /* is_restore = */ false, /* on_cluster = */ false, allow_concurrent_backup_, concurrency_counters_) + , concurrency_check(/* is_restore = */ false, /* on_cluster = */ false, /* zookeeper_path = */ "", allow_concurrent_backup_, concurrency_counters_) , file_infos(is_plain_backup_) { } diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 09991c0d301..e63fcde981a 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -23,20 +23,19 @@ class BackupCoordinationLocal : public IBackupCoordination { public: explicit BackupCoordinationLocal( - const UUID & backup_uuid_, bool is_plain_backup_, bool allow_concurrent_backup_, BackupConcurrencyCounters & concurrency_counters_); ~BackupCoordinationLocal() override; + void setBackupQueryIsSentToOtherHosts() override {} + bool isBackupQuerySentToOtherHosts() const override { return false; } Strings setStage(const String &, const String &, bool) override { return {}; } - void setBackupQueryWasSentToOtherHosts() override {} - bool trySetError(std::exception_ptr) override { return true; } - void finish() override {} - bool tryFinishAfterError() noexcept override { return true; } - void waitForOtherHostsToFinish() override {} - bool tryWaitForOtherHostsToFinishAfterError() noexcept override { return true; } + bool setError(std::exception_ptr, bool) override { return true; } + bool waitOtherHostsFinish(bool) const override { return true; } + bool finish(bool) override { return true; } + bool cleanup(bool) override { return true; } void addReplicatedPartNames(const String & table_zk_path, const String & table_name_for_logs, const String & replica_name, const std::vector & part_names_and_checksums) override; diff --git a/src/Backups/BackupCoordinationOnCluster.cpp b/src/Backups/BackupCoordinationOnCluster.cpp index dc34939f805..1b14f226eff 100644 --- a/src/Backups/BackupCoordinationOnCluster.cpp +++ b/src/Backups/BackupCoordinationOnCluster.cpp @@ -184,17 +184,21 @@ BackupCoordinationOnCluster::BackupCoordinationOnCluster( , plain_backup(is_plain_backup_) , log(getLogger("BackupCoordinationOnCluster")) , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); }) - , concurrency_check(backup_uuid_, /* is_restore = */ false, /* on_cluster = */ true, allow_concurrent_backup_, concurrency_counters_) - , stage_sync(/* is_restore = */ false, fs::path{zookeeper_path} / "stage", current_host, all_hosts, allow_concurrent_backup_, with_retries, schedule_, process_list_element_, log) - , cleaner(zookeeper_path, with_retries, log) + , cleaner(/* is_restore = */ false, zookeeper_path, with_retries, log) + , stage_sync(/* is_restore = */ false, fs::path{zookeeper_path} / "stage", current_host, all_hosts, allow_concurrent_backup_, concurrency_counters_, with_retries, schedule_, process_list_element_, log) { - createRootNodes(); + try + { + createRootNodes(); + } + catch (...) + { + stage_sync.setError(std::current_exception(), /* throw_if_error = */ false); + throw; + } } -BackupCoordinationOnCluster::~BackupCoordinationOnCluster() -{ - tryFinishImpl(); -} +BackupCoordinationOnCluster::~BackupCoordinationOnCluster() = default; void BackupCoordinationOnCluster::createRootNodes() { @@ -217,69 +221,52 @@ void BackupCoordinationOnCluster::createRootNodes() }); } +void BackupCoordinationOnCluster::setBackupQueryIsSentToOtherHosts() +{ + stage_sync.setQueryIsSentToOtherHosts(); +} + +bool BackupCoordinationOnCluster::isBackupQuerySentToOtherHosts() const +{ + return stage_sync.isQuerySentToOtherHosts(); +} + Strings BackupCoordinationOnCluster::setStage(const String & new_stage, const String & message, bool sync) { stage_sync.setStage(new_stage, message); - - if (!sync) - return {}; - - return stage_sync.waitForHostsToReachStage(new_stage, all_hosts_without_initiator); + if (sync) + return stage_sync.waitHostsReachStage(all_hosts_without_initiator, new_stage); + return {}; } -void BackupCoordinationOnCluster::setBackupQueryWasSentToOtherHosts() +bool BackupCoordinationOnCluster::setError(std::exception_ptr exception, bool throw_if_error) { - backup_query_was_sent_to_other_hosts = true; + return stage_sync.setError(exception, throw_if_error); } -bool BackupCoordinationOnCluster::trySetError(std::exception_ptr exception) +bool BackupCoordinationOnCluster::waitOtherHostsFinish(bool throw_if_error) const { - return stage_sync.trySetError(exception); + return stage_sync.waitOtherHostsFinish(throw_if_error); } -void BackupCoordinationOnCluster::finish() +bool BackupCoordinationOnCluster::finish(bool throw_if_error) { - bool other_hosts_also_finished = false; - stage_sync.finish(other_hosts_also_finished); - - if ((current_host == kInitiator) && (other_hosts_also_finished || !backup_query_was_sent_to_other_hosts)) - cleaner.cleanup(); + return stage_sync.finish(throw_if_error); } -bool BackupCoordinationOnCluster::tryFinishAfterError() noexcept +bool BackupCoordinationOnCluster::cleanup(bool throw_if_error) { - return tryFinishImpl(); -} - -bool BackupCoordinationOnCluster::tryFinishImpl() noexcept -{ - bool other_hosts_also_finished = false; - if (!stage_sync.tryFinishAfterError(other_hosts_also_finished)) - return false; - - if ((current_host == kInitiator) && (other_hosts_also_finished || !backup_query_was_sent_to_other_hosts)) + /// All the hosts must finish before we remove the coordination nodes. + bool expect_other_hosts_finished = stage_sync.isQuerySentToOtherHosts() || !stage_sync.isErrorSet(); + bool all_hosts_finished = stage_sync.finished() && (stage_sync.otherHostsFinished() || !expect_other_hosts_finished); + if (!all_hosts_finished) { - if (!cleaner.tryCleanupAfterError()) - return false; - } - - return true; -} - -void BackupCoordinationOnCluster::waitForOtherHostsToFinish() -{ - if ((current_host != kInitiator) || !backup_query_was_sent_to_other_hosts) - return; - stage_sync.waitForOtherHostsToFinish(); -} - -bool BackupCoordinationOnCluster::tryWaitForOtherHostsToFinishAfterError() noexcept -{ - if (current_host != kInitiator) + auto unfinished_hosts = expect_other_hosts_finished ? stage_sync.getUnfinishedHosts() : Strings{current_host}; + LOG_INFO(log, "Skipping removing nodes from ZooKeeper because hosts {} didn't finish", + BackupCoordinationStageSync::getHostsDesc(unfinished_hosts)); return false; - if (!backup_query_was_sent_to_other_hosts) - return true; - return stage_sync.tryWaitForOtherHostsToFinishAfterError(); + } + return cleaner.cleanup(throw_if_error); } ZooKeeperRetriesInfo BackupCoordinationOnCluster::getOnClusterInitializationKeeperRetriesInfo() const diff --git a/src/Backups/BackupCoordinationOnCluster.h b/src/Backups/BackupCoordinationOnCluster.h index 7369c2cc746..b439ab619d8 100644 --- a/src/Backups/BackupCoordinationOnCluster.h +++ b/src/Backups/BackupCoordinationOnCluster.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -20,7 +19,7 @@ class BackupCoordinationOnCluster : public IBackupCoordination { public: /// Empty string as the current host is used to mark the initiator of a BACKUP ON CLUSTER query. - static const constexpr std::string_view kInitiator; + static const constexpr std::string_view kInitiator = BackupCoordinationStageSync::kInitiator; BackupCoordinationOnCluster( const UUID & backup_uuid_, @@ -37,13 +36,13 @@ public: ~BackupCoordinationOnCluster() override; + void setBackupQueryIsSentToOtherHosts() override; + bool isBackupQuerySentToOtherHosts() const override; Strings setStage(const String & new_stage, const String & message, bool sync) override; - void setBackupQueryWasSentToOtherHosts() override; - bool trySetError(std::exception_ptr exception) override; - void finish() override; - bool tryFinishAfterError() noexcept override; - void waitForOtherHostsToFinish() override; - bool tryWaitForOtherHostsToFinishAfterError() noexcept override; + bool setError(std::exception_ptr exception, bool throw_if_error) override; + bool waitOtherHostsFinish(bool throw_if_error) const override; + bool finish(bool throw_if_error) override; + bool cleanup(bool throw_if_error) override; void addReplicatedPartNames( const String & table_zk_path, @@ -110,11 +109,10 @@ private: const bool plain_backup; LoggerPtr const log; + /// The order is important: `stage_sync` must be initialized after `with_retries` and `cleaner`. const WithRetries with_retries; - BackupConcurrencyCheck concurrency_check; - BackupCoordinationStageSync stage_sync; BackupCoordinationCleaner cleaner; - std::atomic backup_query_was_sent_to_other_hosts = false; + BackupCoordinationStageSync stage_sync; mutable std::optional replicated_tables TSA_GUARDED_BY(replicated_tables_mutex); mutable std::optional replicated_access TSA_GUARDED_BY(replicated_access_mutex); diff --git a/src/Backups/BackupCoordinationStageSync.cpp b/src/Backups/BackupCoordinationStageSync.cpp index 9a05f9490c2..fcf09d7c315 100644 --- a/src/Backups/BackupCoordinationStageSync.cpp +++ b/src/Backups/BackupCoordinationStageSync.cpp @@ -42,9 +42,6 @@ namespace kCurrentVersion = 2, }; - - /// Empty string as the current host is used to mark the initiator of a BACKUP ON CLUSTER or RESTORE ON CLUSTER query. - const constexpr std::string_view kInitiator; } bool BackupCoordinationStageSync::HostInfo::operator ==(const HostInfo & other) const @@ -63,12 +60,32 @@ bool BackupCoordinationStageSync::State::operator ==(const State & other) const bool BackupCoordinationStageSync::State::operator !=(const State & other) const = default; +void BackupCoordinationStageSync::State::merge(const State & other) +{ + if (other.host_with_error && !host_with_error) + { + const String & host = *other.host_with_error; + host_with_error = host; + hosts.at(host).exception = other.hosts.at(host).exception; + } + + for (const auto & [host, other_host_info] : other.hosts) + { + auto & host_info = hosts.at(host); + host_info.stages.insert(other_host_info.stages.begin(), other_host_info.stages.end()); + if (other_host_info.finished) + host_info.finished = true; + } +} + + BackupCoordinationStageSync::BackupCoordinationStageSync( bool is_restore_, const String & zookeeper_path_, const String & current_host_, const Strings & all_hosts_, bool allow_concurrency_, + BackupConcurrencyCounters & concurrency_counters_, const WithRetries & with_retries_, ThreadPoolCallbackRunnerUnsafe schedule_, QueryStatusPtr process_list_element_, @@ -89,35 +106,29 @@ BackupCoordinationStageSync::BackupCoordinationStageSync( , max_attempts_after_bad_version(with_retries.getKeeperSettings().max_attempts_after_bad_version) , zookeeper_path(zookeeper_path_) , root_zookeeper_path(zookeeper_path.parent_path().parent_path()) - , operation_node_path(zookeeper_path.parent_path()) + , operation_zookeeper_path(zookeeper_path.parent_path()) , operation_node_name(zookeeper_path.parent_path().filename()) - , stage_node_path(zookeeper_path) , start_node_path(zookeeper_path / ("started|" + current_host)) , finish_node_path(zookeeper_path / ("finished|" + current_host)) , num_hosts_node_path(zookeeper_path / "num_hosts") + , error_node_path(zookeeper_path / "error") , alive_node_path(zookeeper_path / ("alive|" + current_host)) , alive_tracker_node_path(fs::path{root_zookeeper_path} / "alive_tracker") - , error_node_path(zookeeper_path / "error") , zk_nodes_changed(std::make_shared()) { - if ((zookeeper_path.filename() != "stage") || !operation_node_name.starts_with(is_restore ? "restore-" : "backup-") - || (root_zookeeper_path == operation_node_path)) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected path in ZooKeeper specified: {}", zookeeper_path); - } - initializeState(); createRootNodes(); try { + concurrency_check.emplace(is_restore, /* on_cluster = */ true, zookeeper_path, allow_concurrency, concurrency_counters_); createStartAndAliveNodes(); startWatchingThread(); } catch (...) { - trySetError(std::current_exception()); - tryFinishImpl(); + if (setError(std::current_exception(), /* throw_if_error = */ false)) + finish(/* throw_if_error = */ false); throw; } } @@ -125,7 +136,26 @@ BackupCoordinationStageSync::BackupCoordinationStageSync( BackupCoordinationStageSync::~BackupCoordinationStageSync() { - tryFinishImpl(); + /// Normally either finish() or setError() must be called. + if (!tried_to_finish) + { + if (state.host_with_error) + { + /// setError() was called and succeeded. + finish(/* throw_if_error = */ false); + } + else if (!tried_to_set_error) + { + /// Neither finish() nor setError() were called, it's a bug. + chassert(false, "~BackupCoordinationStageSync() is called without finish() or setError()"); + LOG_ERROR(log, "~BackupCoordinationStageSync() is called without finish() or setError()"); + } + } + + /// Normally the watching thread should be stopped already because the finish() function stops it. + /// However if an error happened then the watching thread can be still running, + /// so here in the destructor we have to ensure that it's stopped. + stopWatchingThread(); } @@ -137,6 +167,12 @@ void BackupCoordinationStageSync::initializeState() for (const String & host : all_hosts) state.hosts.emplace(host, HostInfo{.host = host, .last_connection_time = now, .last_connection_time_monotonic = monotonic_now}); + + if (!state.hosts.contains(current_host)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "List of hosts must contain the current host"); + + if (!state.hosts.contains(String{kInitiator})) + throw Exception(ErrorCodes::LOGICAL_ERROR, "List of hosts must contain the initiator"); } @@ -179,6 +215,12 @@ String BackupCoordinationStageSync::getHostsDesc(const Strings & hosts) void BackupCoordinationStageSync::createRootNodes() { + if ((zookeeper_path.filename() != "stage") || !operation_node_name.starts_with(is_restore ? "restore-" : "backup-") + || (root_zookeeper_path == operation_zookeeper_path)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected path in ZooKeeper specified: {}", zookeeper_path); + } + auto holder = with_retries.createRetriesControlHolder("BackupStageSync::createRootNodes", WithRetries::kInitialization); holder.retries_ctl.retryLoop( [&, &zookeeper = holder.faulty_zookeeper]() @@ -252,27 +294,27 @@ void BackupCoordinationStageSync::createStartAndAliveNodes(Coordination::ZooKeep Coordination::Requests requests; requests.reserve(6); - size_t operation_node_path_pos = static_cast(-1); - if (!zookeeper->exists(operation_node_path)) + size_t operation_node_pos = static_cast(-1); + if (!zookeeper->exists(operation_zookeeper_path)) { - operation_node_path_pos = requests.size(); - requests.emplace_back(zkutil::makeCreateRequest(operation_node_path, "", zkutil::CreateMode::Persistent)); + operation_node_pos = requests.size(); + requests.emplace_back(zkutil::makeCreateRequest(operation_zookeeper_path, "", zkutil::CreateMode::Persistent)); } - size_t stage_node_path_pos = static_cast(-1); - if (!zookeeper->exists(stage_node_path)) + size_t zookeeper_path_pos = static_cast(-1); + if (!zookeeper->exists(zookeeper_path)) { - stage_node_path_pos = requests.size(); - requests.emplace_back(zkutil::makeCreateRequest(stage_node_path, "", zkutil::CreateMode::Persistent)); + zookeeper_path_pos = requests.size(); + requests.emplace_back(zkutil::makeCreateRequest(zookeeper_path, "", zkutil::CreateMode::Persistent)); } - size_t num_hosts_node_path_pos = requests.size(); + size_t num_hosts_node_pos = requests.size(); if (num_hosts) requests.emplace_back(zkutil::makeSetRequest(num_hosts_node_path, toString(*num_hosts + 1), num_hosts_version)); else requests.emplace_back(zkutil::makeCreateRequest(num_hosts_node_path, "1", zkutil::CreateMode::Persistent)); - size_t alive_tracker_node_path_pos = requests.size(); + size_t alive_tracker_node_pos = requests.size(); requests.emplace_back(zkutil::makeSetRequest(alive_tracker_node_path, "", alive_tracker_version)); requests.emplace_back(zkutil::makeCreateRequest(start_node_path, std::to_string(kCurrentVersion), zkutil::CreateMode::Persistent)); @@ -284,7 +326,7 @@ void BackupCoordinationStageSync::createStartAndAliveNodes(Coordination::ZooKeep if (code == Coordination::Error::ZOK) { LOG_INFO(log, "Created start node #{} in ZooKeeper for {} (coordination version: {})", - num_hosts.value_or(0) + 1, current_host_desc, kCurrentVersion); + num_hosts.value_or(0) + 1, current_host_desc, static_cast(kCurrentVersion)); return; } @@ -294,40 +336,34 @@ void BackupCoordinationStageSync::createStartAndAliveNodes(Coordination::ZooKeep LOG_TRACE(log, "{} (attempt #{}){}", message, attempt_no, will_try_again ? ", will try again" : ""); }; - if ((responses.size() > operation_node_path_pos) && - (responses[operation_node_path_pos]->error == Coordination::Error::ZNODEEXISTS)) + if ((operation_node_pos < responses.size()) && + (responses[operation_node_pos]->error == Coordination::Error::ZNODEEXISTS)) { - show_error_before_next_attempt(fmt::format("Node {} in ZooKeeper already exists", operation_node_path)); + show_error_before_next_attempt(fmt::format("Node {} already exists", operation_zookeeper_path)); /// needs another attempt } - else if ((responses.size() > stage_node_path_pos) && - (responses[stage_node_path_pos]->error == Coordination::Error::ZNODEEXISTS)) + else if ((zookeeper_path_pos < responses.size()) && + (responses[zookeeper_path_pos]->error == Coordination::Error::ZNODEEXISTS)) { - show_error_before_next_attempt(fmt::format("Node {} in ZooKeeper already exists", stage_node_path)); + show_error_before_next_attempt(fmt::format("Node {} already exists", zookeeper_path)); /// needs another attempt } - else if ((responses.size() > num_hosts_node_path_pos) && num_hosts && - (responses[num_hosts_node_path_pos]->error == Coordination::Error::ZBADVERSION)) + else if ((num_hosts_node_pos < responses.size()) && !num_hosts && + (responses[num_hosts_node_pos]->error == Coordination::Error::ZNODEEXISTS)) { - show_error_before_next_attempt("Other host changed the 'num_hosts' node in ZooKeeper"); + show_error_before_next_attempt(fmt::format("Node {} already exists", num_hosts_node_path)); + /// needs another attempt + } + else if ((num_hosts_node_pos < responses.size()) && num_hosts && + (responses[num_hosts_node_pos]->error == Coordination::Error::ZBADVERSION)) + { + show_error_before_next_attempt(fmt::format("The version of node {} changed", num_hosts_node_path)); num_hosts.reset(); /// needs to reread 'num_hosts' again } - else if ((responses.size() > num_hosts_node_path_pos) && num_hosts && - (responses[num_hosts_node_path_pos]->error == Coordination::Error::ZNONODE)) + else if ((alive_tracker_node_pos < responses.size()) && + (responses[alive_tracker_node_pos]->error == Coordination::Error::ZBADVERSION)) { - show_error_before_next_attempt("Other host removed the 'num_hosts' node in ZooKeeper"); - num_hosts.reset(); /// needs to reread 'num_hosts' again - } - else if ((responses.size() > num_hosts_node_path_pos) && !num_hosts && - (responses[num_hosts_node_path_pos]->error == Coordination::Error::ZNODEEXISTS)) - { - show_error_before_next_attempt("Other host created the 'num_hosts' node in ZooKeeper"); - /// needs another attempt - } - else if ((responses.size() > alive_tracker_node_path_pos) && - (responses[alive_tracker_node_path_pos]->error == Coordination::Error::ZBADVERSION)) - { - show_error_before_next_attempt("Concurrent backup or restore changed some 'alive' nodes in ZooKeeper"); + show_error_before_next_attempt(fmt::format("The version of node {} changed", alive_tracker_node_path)); check_concurrency = true; /// needs to recheck for concurrency again } else @@ -337,8 +373,7 @@ void BackupCoordinationStageSync::createStartAndAliveNodes(Coordination::ZooKeep } throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, - "Couldn't create the 'start' node in ZooKeeper for {} after {} attempts", - current_host_desc, max_attempts_after_bad_version); + "Couldn't create node {} in ZooKeeper after {} attempts", start_node_path, max_attempts_after_bad_version); } @@ -387,36 +422,53 @@ void BackupCoordinationStageSync::startWatchingThread() void BackupCoordinationStageSync::stopWatchingThread() { - should_stop_watching_thread = true; + { + std::lock_guard lock{mutex}; + if (should_stop_watching_thread) + return; + should_stop_watching_thread = true; - /// Wake up waiting threads. - if (zk_nodes_changed) - zk_nodes_changed->set(); - state_changed.notify_all(); + /// Wake up waiting threads. + if (zk_nodes_changed) + zk_nodes_changed->set(); + state_changed.notify_all(); + } if (watching_thread_future.valid()) watching_thread_future.wait(); + + LOG_TRACE(log, "Stopped the watching thread"); } void BackupCoordinationStageSync::watchingThread() { - while (!should_stop_watching_thread) + auto should_stop = [&] + { + std::lock_guard lock{mutex}; + return should_stop_watching_thread; + }; + + while (!should_stop()) { try { /// Check if the current BACKUP or RESTORE command is already cancelled. checkIfQueryCancelled(); + } + catch (...) + { + tryLogCurrentException(log, "Caugth exception while watching"); + } - /// Reset the `connected` flag for each host, we'll set them to true again after we find the 'alive' nodes. - resetConnectedFlag(); - + try + { /// Recreate the 'alive' node if necessary and read a new state from ZooKeeper. auto holder = with_retries.createRetriesControlHolder("BackupStageSync::watchingThread"); auto & zookeeper = holder.faulty_zookeeper; with_retries.renewZooKeeper(zookeeper); - if (should_stop_watching_thread) + if (should_stop()) return; /// Recreate the 'alive' node if it was removed. @@ -427,7 +479,10 @@ void BackupCoordinationStageSync::watchingThread() } catch (...) { - tryLogCurrentException(log, "Caugth exception while watching"); + tryLogCurrentException(log, "Caught exception while watching"); + + /// Reset the `connected` flag for each host, we'll set them to true again after we find the 'alive' nodes. + resetConnectedFlag(); } try @@ -438,7 +493,7 @@ void BackupCoordinationStageSync::watchingThread() } catch (...) { - tryLogCurrentException(log, "Caugth exception while checking if the query should be cancelled"); + tryLogCurrentException(log, "Caught exception while watching"); } zk_nodes_changed->tryWait(sync_period_ms.count()); @@ -473,7 +528,7 @@ void BackupCoordinationStageSync::readCurrentState(Coordination::ZooKeeperWithFa zk_nodes_changed->reset(); /// Get zk nodes and subscribe on their changes. - Strings new_zk_nodes = zookeeper->getChildren(stage_node_path, nullptr, zk_nodes_changed); + Strings new_zk_nodes = zookeeper->getChildren(zookeeper_path, nullptr, zk_nodes_changed); std::sort(new_zk_nodes.begin(), new_zk_nodes.end()); /// Sorting is necessary because we compare the list of zk nodes with its previous versions. State new_state; @@ -492,6 +547,8 @@ void BackupCoordinationStageSync::readCurrentState(Coordination::ZooKeeperWithFa zk_nodes = new_zk_nodes; new_state = state; + for (auto & [_, host_info] : new_state.hosts) + host_info.connected = false; } auto get_host_info = [&](const String & host) -> HostInfo * @@ -514,7 +571,8 @@ void BackupCoordinationStageSync::readCurrentState(Coordination::ZooKeeperWithFa { String serialized_error = zookeeper->get(error_node_path); auto [exception, host] = parseErrorNode(serialized_error); - if (auto * host_info = get_host_info(host)) + auto * host_info = get_host_info(host); + if (exception && host_info) { host_info->exception = exception; new_state.host_with_error = host; @@ -576,6 +634,9 @@ void BackupCoordinationStageSync::readCurrentState(Coordination::ZooKeeperWithFa { std::lock_guard lock{mutex}; + /// We were reading `new_state` from ZooKeeper with `mutex` unlocked, so `state` could get more information during that reading, + /// we don't want to lose that information, that's why we use merge() here. + new_state.merge(state); was_state_changed = (new_state != state); state = std::move(new_state); } @@ -604,26 +665,10 @@ int BackupCoordinationStageSync::parseStartNode(const String & start_node_conten } -std::pair BackupCoordinationStageSync::parseErrorNode(const String & error_node_contents) -{ - ReadBufferFromOwnString buf{error_node_contents}; - String host; - readStringBinary(host, buf); - auto exception = std::make_exception_ptr(readException(buf, fmt::format("Got error from {}", getHostDesc(host)))); - return {exception, host}; -} - - void BackupCoordinationStageSync::checkIfQueryCancelled() { if (process_list_element->checkTimeLimitSoft()) return; /// Not cancelled. - - std::lock_guard lock{mutex}; - if (state.cancelled) - return; /// Already marked as cancelled. - - state.cancelled = true; state_changed.notify_all(); } @@ -634,13 +679,13 @@ void BackupCoordinationStageSync::cancelQueryIfError() { std::lock_guard lock{mutex}; - if (state.cancelled || !state.host_with_error) + if (!state.host_with_error) return; - state.cancelled = true; exception = state.hosts.at(*state.host_with_error).exception; } + chassert(exception); process_list_element->cancelQuery(false, exception); state_changed.notify_all(); } @@ -652,7 +697,7 @@ void BackupCoordinationStageSync::cancelQueryIfDisconnectedTooLong() { std::lock_guard lock{mutex}; - if (state.cancelled || state.host_with_error || ((failure_after_host_disconnected_for_seconds.count() == 0))) + if (state.host_with_error || ((failure_after_host_disconnected_for_seconds.count() == 0))) return; auto monotonic_now = std::chrono::steady_clock::now(); @@ -685,27 +730,92 @@ void BackupCoordinationStageSync::cancelQueryIfDisconnectedTooLong() } } } - - if (!exception) - return; - - state.cancelled = true; } + if (!exception) + return; + process_list_element->cancelQuery(false, exception); state_changed.notify_all(); } +void BackupCoordinationStageSync::setQueryIsSentToOtherHosts() +{ + std::lock_guard lock{mutex}; + query_is_sent_to_other_hosts = true; +} + +bool BackupCoordinationStageSync::isQuerySentToOtherHosts() const +{ + std::lock_guard lock{mutex}; + return query_is_sent_to_other_hosts; +} + + void BackupCoordinationStageSync::setStage(const String & stage, const String & stage_result) { LOG_INFO(log, "{} reached stage {}", current_host_desc, stage); + + { + std::lock_guard lock{mutex}; + if (state.hosts.at(current_host).stages.contains(stage)) + return; /// Already set. + } + + if ((getInitiatorVersion() == kVersionWithoutFinishNode) && (stage == BackupCoordinationStage::COMPLETED)) + { + LOG_TRACE(log, "Stopping the watching thread because the initiator uses outdated version {}", getInitiatorVersion()); + stopWatchingThread(); + } + auto holder = with_retries.createRetriesControlHolder("BackupStageSync::setStage"); holder.retries_ctl.retryLoop([&, &zookeeper = holder.faulty_zookeeper]() { with_retries.renewZooKeeper(zookeeper); - zookeeper->createIfNotExists(getStageNodePath(stage), stage_result); + createStageNode(stage, stage_result, zookeeper); }); + + /// If the initiator of the query has that old version then it doesn't expect us to create the 'finish' node and moreover + /// the initiator can start removing all the nodes immediately after all hosts report about reaching the "completed" status. + /// So to avoid weird errors in the logs we won't create the 'finish' node if the initiator of the query has that old version. + if ((getInitiatorVersion() == kVersionWithoutFinishNode) && (stage == BackupCoordinationStage::COMPLETED)) + { + LOG_INFO(log, "Skipped creating the 'finish' node because the initiator uses outdated version {}", getInitiatorVersion()); + std::lock_guard lock{mutex}; + tried_to_finish = true; + state.hosts.at(current_host).finished = true; + } +} + + +void BackupCoordinationStageSync::createStageNode(const String & stage, const String & stage_result, Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper) +{ + String serialized_error; + if (zookeeper->tryGet(error_node_path, serialized_error)) + { + auto [exception, host] = parseErrorNode(serialized_error); + if (exception) + std::rethrow_exception(exception); + } + + auto code = zookeeper->tryCreate(getStageNodePath(stage), stage_result, zkutil::CreateMode::Persistent); + if (code == Coordination::Error::ZOK) + { + std::lock_guard lock{mutex}; + state.hosts.at(current_host).stages[stage] = stage_result; + return; + } + + if (code == Coordination::Error::ZNODEEXISTS) + { + String another_result = zookeeper->get(getStageNodePath(stage)); + std::lock_guard lock{mutex}; + state.hosts.at(current_host).stages[stage] = another_result; + return; + } + + throw zkutil::KeeperException::fromPath(code, getStageNodePath(stage)); } @@ -715,71 +825,7 @@ String BackupCoordinationStageSync::getStageNodePath(const String & stage) const } -bool BackupCoordinationStageSync::trySetError(std::exception_ptr exception) noexcept -{ - try - { - std::rethrow_exception(exception); - } - catch (const Exception & e) - { - return trySetError(e); - } - catch (...) - { - return trySetError(Exception(getCurrentExceptionMessageAndPattern(true, true), getCurrentExceptionCode())); - } -} - - -bool BackupCoordinationStageSync::trySetError(const Exception & exception) -{ - try - { - setError(exception); - return true; - } - catch (...) - { - return false; - } -} - - -void BackupCoordinationStageSync::setError(const Exception & exception) -{ - /// Most likely this exception has been already logged so here we're logging it without stacktrace. - String exception_message = getExceptionMessage(exception, /* with_stacktrace= */ false, /* check_embedded_stacktrace= */ true); - LOG_INFO(log, "Sending exception from {} to other hosts: {}", current_host_desc, exception_message); - - auto holder = with_retries.createRetriesControlHolder("BackupStageSync::setError", WithRetries::kErrorHandling); - - holder.retries_ctl.retryLoop([&, &zookeeper = holder.faulty_zookeeper]() - { - with_retries.renewZooKeeper(zookeeper); - - WriteBufferFromOwnString buf; - writeStringBinary(current_host, buf); - writeException(exception, buf, true); - auto code = zookeeper->tryCreate(error_node_path, buf.str(), zkutil::CreateMode::Persistent); - - if (code == Coordination::Error::ZOK) - { - LOG_TRACE(log, "Sent exception from {} to other hosts", current_host_desc); - } - else if (code == Coordination::Error::ZNODEEXISTS) - { - LOG_INFO(log, "An error has been already assigned for this {}", operation_name); - } - else - { - throw zkutil::KeeperException::fromPath(code, error_node_path); - } - }); -} - - -Strings BackupCoordinationStageSync::waitForHostsToReachStage(const String & stage_to_wait, const Strings & hosts, std::optional timeout) const +Strings BackupCoordinationStageSync::waitHostsReachStage(const Strings & hosts, const String & stage_to_wait) const { Strings results; results.resize(hosts.size()); @@ -787,44 +833,28 @@ Strings BackupCoordinationStageSync::waitForHostsToReachStage(const String & sta std::unique_lock lock{mutex}; /// TSA_NO_THREAD_SAFETY_ANALYSIS is here because Clang Thread Safety Analysis doesn't understand std::unique_lock. - auto check_if_hosts_ready = [&](bool time_is_out) TSA_NO_THREAD_SAFETY_ANALYSIS + auto check_if_hosts_reach_stage = [&]() TSA_NO_THREAD_SAFETY_ANALYSIS { - return checkIfHostsReachStage(hosts, stage_to_wait, time_is_out, timeout, results); + return checkIfHostsReachStage(hosts, stage_to_wait, results); }; - if (timeout) - { - if (!state_changed.wait_for(lock, *timeout, [&] { return check_if_hosts_ready(/* time_is_out = */ false); })) - check_if_hosts_ready(/* time_is_out = */ true); - } - else - { - state_changed.wait(lock, [&] { return check_if_hosts_ready(/* time_is_out = */ false); }); - } + state_changed.wait(lock, check_if_hosts_reach_stage); return results; } -bool BackupCoordinationStageSync::checkIfHostsReachStage( - const Strings & hosts, - const String & stage_to_wait, - bool time_is_out, - std::optional timeout, - Strings & results) const +bool BackupCoordinationStageSync::checkIfHostsReachStage(const Strings & hosts, const String & stage_to_wait, Strings & results) const { - if (should_stop_watching_thread) - throw Exception(ErrorCodes::LOGICAL_ERROR, "finish() was called while waiting for a stage"); - process_list_element->checkTimeLimit(); for (size_t i = 0; i != hosts.size(); ++i) { const String & host = hosts[i]; auto it = state.hosts.find(host); - if (it == state.hosts.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "waitForHostsToReachStage() was called for unexpected {}, all hosts are {}", getHostDesc(host), getHostsDesc(all_hosts)); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "waitHostsReachStage() was called for unexpected {}, all hosts are {}", getHostDesc(host), getHostsDesc(all_hosts)); const HostInfo & host_info = it->second; auto stage_it = host_info.stages.find(stage_to_wait); @@ -835,10 +865,11 @@ bool BackupCoordinationStageSync::checkIfHostsReachStage( } if (host_info.finished) - { throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "{} finished without coming to stage {}", getHostDesc(host), stage_to_wait); - } + + if (should_stop_watching_thread) + throw Exception(ErrorCodes::LOGICAL_ERROR, "waitHostsReachStage() can't wait for stage {} after the watching thread stopped", stage_to_wait); String host_status; if (!host_info.started) @@ -846,85 +877,73 @@ bool BackupCoordinationStageSync::checkIfHostsReachStage( else if (!host_info.connected) host_status = fmt::format(": the host is currently disconnected, last connection was at {}", host_info.last_connection_time); - if (!time_is_out) - { - LOG_TRACE(log, "Waiting for {} to reach stage {}{}", getHostDesc(host), stage_to_wait, host_status); - return false; - } - else - { - throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, - "Waited longer than timeout {} for {} to reach stage {}{}", - *timeout, getHostDesc(host), stage_to_wait, host_status); - } + LOG_TRACE(log, "Waiting for {} to reach stage {}{}", getHostDesc(host), stage_to_wait, host_status); + return false; /// wait for next change of `state_changed` } LOG_INFO(log, "Hosts {} reached stage {}", getHostsDesc(hosts), stage_to_wait); - return true; + return true; /// stop waiting } -void BackupCoordinationStageSync::finish(bool & other_hosts_also_finished) +bool BackupCoordinationStageSync::finish(bool throw_if_error) { - tryFinishImpl(other_hosts_also_finished, /* throw_if_error = */ true, /* retries_kind = */ WithRetries::kNormal); + WithRetries::Kind retries_kind = WithRetries::kNormal; + if (throw_if_error) + retries_kind = WithRetries::kErrorHandling; + + return finishImpl(throw_if_error, retries_kind); } -bool BackupCoordinationStageSync::tryFinishAfterError(bool & other_hosts_also_finished) noexcept +bool BackupCoordinationStageSync::finishImpl(bool throw_if_error, WithRetries::Kind retries_kind) { - return tryFinishImpl(other_hosts_also_finished, /* throw_if_error = */ false, /* retries_kind = */ WithRetries::kErrorHandling); -} - - -bool BackupCoordinationStageSync::tryFinishImpl() -{ - bool other_hosts_also_finished; - return tryFinishAfterError(other_hosts_also_finished); -} - - -bool BackupCoordinationStageSync::tryFinishImpl(bool & other_hosts_also_finished, bool throw_if_error, WithRetries::Kind retries_kind) -{ - auto get_value_other_hosts_also_finished = [&] TSA_REQUIRES(mutex) - { - other_hosts_also_finished = true; - for (const auto & [host, host_info] : state.hosts) - { - if ((host != current_host) && !host_info.finished) - other_hosts_also_finished = false; - } - }; - { std::lock_guard lock{mutex}; - if (finish_result.succeeded) + + if (finishedNoLock()) { - get_value_other_hosts_also_finished(); + LOG_INFO(log, "The finish node for {} already exists", current_host_desc); return true; } - if (finish_result.exception) + + if (tried_to_finish) { - if (throw_if_error) - std::rethrow_exception(finish_result.exception); + /// We don't repeat creating the finish node, no matter if it was successful or not. + LOG_INFO(log, "Skipped creating the finish node for {} because earlier we failed to do that", current_host_desc); return false; } + + bool failed_to_set_error = tried_to_set_error && !state.host_with_error; + if (failed_to_set_error) + { + /// Tried to create the 'error' node, but failed. + /// Then it's better not to create the 'finish' node in this case because otherwise other hosts might think we've succeeded. + LOG_INFO(log, "Skipping creating the finish node for {} because there was an error which we were unable to send to other hosts", current_host_desc); + return false; + } + + if (current_host == kInitiator) + { + /// Normally the initiator should wait for other hosts to finish before creating its own finish node. + /// We show warning if some of the other hosts didn't finish. + bool expect_other_hosts_finished = query_is_sent_to_other_hosts || !state.host_with_error; + bool other_hosts_finished = otherHostsFinishedNoLock() || !expect_other_hosts_finished; + if (!other_hosts_finished) + LOG_WARNING(log, "Hosts {} didn't finish before the initiator", getHostsDesc(getUnfinishedOtherHostsNoLock())); + } } + stopWatchingThread(); + try { - stopWatchingThread(); - auto holder = with_retries.createRetriesControlHolder("BackupStageSync::finish", retries_kind); holder.retries_ctl.retryLoop([&, &zookeeper = holder.faulty_zookeeper]() { with_retries.renewZooKeeper(zookeeper); - createFinishNodeAndRemoveAliveNode(zookeeper); + createFinishNodeAndRemoveAliveNode(zookeeper, throw_if_error); }); - - std::lock_guard lock{mutex}; - finish_result.succeeded = true; - get_value_other_hosts_also_finished(); - return true; } catch (...) { @@ -933,63 +952,87 @@ bool BackupCoordinationStageSync::tryFinishImpl(bool & other_hosts_also_finished getCurrentExceptionMessage(/* with_stacktrace= */ false, /* check_embedded_stacktrace= */ true)); std::lock_guard lock{mutex}; - finish_result.exception = std::current_exception(); + tried_to_finish = true; + if (throw_if_error) throw; return false; } + + { + std::lock_guard lock{mutex}; + tried_to_finish = true; + state.hosts.at(current_host).finished = true; + } + + return true; } -void BackupCoordinationStageSync::createFinishNodeAndRemoveAliveNode(Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper) +void BackupCoordinationStageSync::createFinishNodeAndRemoveAliveNode(Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper, bool throw_if_error) { - if (zookeeper->exists(finish_node_path)) - return; - - /// If the initiator of the query has that old version then it doesn't expect us to create the 'finish' node and moreover - /// the initiator can start removing all the nodes immediately after all hosts report about reaching the "completed" status. - /// So to avoid weird errors in the logs we won't create the 'finish' node if the initiator of the query has that old version. - if ((getInitiatorVersion() == kVersionWithoutFinishNode) && (current_host != kInitiator)) - { - LOG_INFO(log, "Skipped creating the 'finish' node because the initiator uses outdated version {}", getInitiatorVersion()); - return; - } - std::optional num_hosts; int num_hosts_version = -1; for (size_t attempt_no = 1; attempt_no <= max_attempts_after_bad_version; ++attempt_no) { + /// The 'num_hosts' node may not exist if createStartAndAliveNodes() failed in the constructor. if (!num_hosts) { + String num_hosts_str; Coordination::Stat stat; - num_hosts = parseFromString(zookeeper->get(num_hosts_node_path, &stat)); - num_hosts_version = stat.version; + if (zookeeper->tryGet(num_hosts_node_path, num_hosts_str, &stat)) + { + num_hosts = parseFromString(num_hosts_str); + num_hosts_version = stat.version; + } } + String serialized_error; + if (throw_if_error && zookeeper->tryGet(error_node_path, serialized_error)) + { + auto [exception, host] = parseErrorNode(serialized_error); + if (exception) + std::rethrow_exception(exception); + } + + if (zookeeper->exists(finish_node_path)) + return; + + bool start_node_exists = zookeeper->exists(start_node_path); + Coordination::Requests requests; requests.reserve(3); requests.emplace_back(zkutil::makeCreateRequest(finish_node_path, "", zkutil::CreateMode::Persistent)); - size_t num_hosts_node_path_pos = requests.size(); - requests.emplace_back(zkutil::makeSetRequest(num_hosts_node_path, toString(*num_hosts - 1), num_hosts_version)); - - size_t alive_node_path_pos = static_cast(-1); + size_t alive_node_pos = static_cast(-1); if (zookeeper->exists(alive_node_path)) { - alive_node_path_pos = requests.size(); + alive_node_pos = requests.size(); requests.emplace_back(zkutil::makeRemoveRequest(alive_node_path, -1)); } + size_t num_hosts_node_pos = static_cast(-1); + if (num_hosts) + { + num_hosts_node_pos = requests.size(); + requests.emplace_back(zkutil::makeSetRequest(num_hosts_node_path, toString(start_node_exists ? (*num_hosts - 1) : *num_hosts), num_hosts_version)); + } + Coordination::Responses responses; auto code = zookeeper->tryMulti(requests, responses); if (code == Coordination::Error::ZOK) { - --*num_hosts; - String hosts_left_desc = ((*num_hosts == 0) ? "no hosts left" : fmt::format("{} hosts left", *num_hosts)); - LOG_INFO(log, "Created the 'finish' node in ZooKeeper for {}, {}", current_host_desc, hosts_left_desc); + String hosts_left_desc; + if (num_hosts) + { + if (start_node_exists) + --*num_hosts; + hosts_left_desc = (*num_hosts == 0) ? ", no hosts left" : fmt::format(", {} hosts left", *num_hosts); + } + LOG_INFO(log, "Created the 'finish' node in ZooKeeper for {}{}", current_host_desc, hosts_left_desc); return; } @@ -999,18 +1042,18 @@ void BackupCoordinationStageSync::createFinishNodeAndRemoveAliveNode(Coordinatio LOG_TRACE(log, "{} (attempt #{}){}", message, attempt_no, will_try_again ? ", will try again" : ""); }; - if ((responses.size() > num_hosts_node_path_pos) && - (responses[num_hosts_node_path_pos]->error == Coordination::Error::ZBADVERSION)) + if ((alive_node_pos < responses.size()) && + (responses[alive_node_pos]->error == Coordination::Error::ZNONODE)) { - show_error_before_next_attempt("Other host changed the 'num_hosts' node in ZooKeeper"); - num_hosts.reset(); /// needs to reread 'num_hosts' again - } - else if ((responses.size() > alive_node_path_pos) && - (responses[alive_node_path_pos]->error == Coordination::Error::ZNONODE)) - { - show_error_before_next_attempt(fmt::format("Node {} in ZooKeeper doesn't exist", alive_node_path_pos)); + show_error_before_next_attempt(fmt::format("Node {} doesn't exist", alive_node_path)); /// needs another attempt } + else if ((num_hosts_node_pos < responses.size()) && + (responses[num_hosts_node_pos]->error == Coordination::Error::ZBADVERSION)) + { + show_error_before_next_attempt(fmt::format("The version of node {} changed", num_hosts_node_path)); + num_hosts.reset(); /// needs to reread 'num_hosts' again + } else { zkutil::KeeperMultiException::check(code, requests, responses); @@ -1026,60 +1069,73 @@ void BackupCoordinationStageSync::createFinishNodeAndRemoveAliveNode(Coordinatio int BackupCoordinationStageSync::getInitiatorVersion() const { std::lock_guard lock{mutex}; - auto it = state.hosts.find(String{kInitiator}); - if (it == state.hosts.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no initiator of this {} query, it's a bug", operation_name); - const HostInfo & host_info = it->second; - return host_info.version; + return state.hosts.at(String{kInitiator}).version; } -void BackupCoordinationStageSync::waitForOtherHostsToFinish() const -{ - tryWaitForOtherHostsToFinishImpl(/* reason = */ "", /* throw_if_error = */ true, /* timeout = */ {}); -} - - -bool BackupCoordinationStageSync::tryWaitForOtherHostsToFinishAfterError() const noexcept +bool BackupCoordinationStageSync::waitOtherHostsFinish(bool throw_if_error) const { std::optional timeout; - if (finish_timeout_after_error.count() != 0) - timeout = finish_timeout_after_error; + String reason; - String reason = fmt::format("{} needs other hosts to finish before cleanup", current_host_desc); - return tryWaitForOtherHostsToFinishImpl(reason, /* throw_if_error = */ false, timeout); + if (!throw_if_error) + { + if (finish_timeout_after_error.count() != 0) + timeout = finish_timeout_after_error; + reason = "after error before cleanup"; + } + + return waitOtherHostsFinishImpl(reason, timeout, throw_if_error); } -bool BackupCoordinationStageSync::tryWaitForOtherHostsToFinishImpl(const String & reason, bool throw_if_error, std::optional timeout) const +bool BackupCoordinationStageSync::waitOtherHostsFinishImpl(const String & reason, std::optional timeout, bool throw_if_error) const { std::unique_lock lock{mutex}; /// TSA_NO_THREAD_SAFETY_ANALYSIS is here because Clang Thread Safety Analysis doesn't understand std::unique_lock. - auto check_if_other_hosts_finish = [&](bool time_is_out) TSA_NO_THREAD_SAFETY_ANALYSIS + auto other_hosts_finished = [&]() TSA_NO_THREAD_SAFETY_ANALYSIS { return otherHostsFinishedNoLock(); }; + + if (other_hosts_finished()) { - return checkIfOtherHostsFinish(reason, throw_if_error, time_is_out, timeout); + LOG_TRACE(log, "Other hosts have already finished"); + return true; + } + + bool failed_to_set_error = TSA_SUPPRESS_WARNING_FOR_READ(tried_to_set_error) && !TSA_SUPPRESS_WARNING_FOR_READ(state).host_with_error; + if (failed_to_set_error) + { + /// Tried to create the 'error' node, but failed. + /// Then it's better not to wait for other hosts to finish in this case because other hosts don't know they should finish. + LOG_INFO(log, "Skipping waiting for other hosts to finish because there was an error which we were unable to send to other hosts"); + return false; + } + + bool result = false; + + /// TSA_NO_THREAD_SAFETY_ANALYSIS is here because Clang Thread Safety Analysis doesn't understand std::unique_lock. + auto check_if_hosts_finish = [&](bool time_is_out) TSA_NO_THREAD_SAFETY_ANALYSIS + { + return checkIfOtherHostsFinish(reason, timeout, time_is_out, result, throw_if_error); }; if (timeout) { - if (state_changed.wait_for(lock, *timeout, [&] { return check_if_other_hosts_finish(/* time_is_out = */ false); })) - return true; - return check_if_other_hosts_finish(/* time_is_out = */ true); + if (!state_changed.wait_for(lock, *timeout, [&] { return check_if_hosts_finish(/* time_is_out = */ false); })) + check_if_hosts_finish(/* time_is_out = */ true); } else { - state_changed.wait(lock, [&] { return check_if_other_hosts_finish(/* time_is_out = */ false); }); - return true; + state_changed.wait(lock, [&] { return check_if_hosts_finish(/* time_is_out = */ false); }); } + + return result; } -bool BackupCoordinationStageSync::checkIfOtherHostsFinish(const String & reason, bool throw_if_error, bool time_is_out, std::optional timeout) const +bool BackupCoordinationStageSync::checkIfOtherHostsFinish( + const String & reason, std::optional timeout, bool time_is_out, bool & result, bool throw_if_error) const { - if (should_stop_watching_thread) - throw Exception(ErrorCodes::LOGICAL_ERROR, "finish() was called while waiting for other hosts to finish"); - if (throw_if_error) process_list_element->checkTimeLimit(); @@ -1088,38 +1144,261 @@ bool BackupCoordinationStageSync::checkIfOtherHostsFinish(const String & reason, if ((host == current_host) || host_info.finished) continue; + String reason_text = reason.empty() ? "" : (" " + reason); + String host_status; if (!host_info.started) host_status = fmt::format(": the host hasn't started working on this {} yet", operation_name); else if (!host_info.connected) host_status = fmt::format(": the host is currently disconnected, last connection was at {}", host_info.last_connection_time); - if (!time_is_out) + if (time_is_out) { - String reason_text = reason.empty() ? "" : (" because " + reason); - LOG_TRACE(log, "Waiting for {} to finish{}{}", getHostDesc(host), reason_text, host_status); - return false; - } - else - { - String reason_text = reason.empty() ? "" : fmt::format(" (reason of waiting: {})", reason); - if (!throw_if_error) - { - LOG_INFO(log, "Waited longer than timeout {} for {} to finish{}{}", - *timeout, getHostDesc(host), host_status, reason_text); - return false; - } - else + if (throw_if_error) { throw Exception(ErrorCodes::FAILED_TO_SYNC_BACKUP_OR_RESTORE, "Waited longer than timeout {} for {} to finish{}{}", - *timeout, getHostDesc(host), host_status, reason_text); + *timeout, getHostDesc(host), reason_text, host_status); } + LOG_INFO(log, "Waited longer than timeout {} for {} to finish{}{}", + *timeout, getHostDesc(host), reason_text, host_status); + result = false; + return true; /// stop waiting } + + if (should_stop_watching_thread) + { + LOG_ERROR(log, "waitOtherHostFinish({}) can't wait for other hosts to finish after the watching thread stopped", throw_if_error); + chassert(false, "waitOtherHostFinish() can't wait for other hosts to finish after the watching thread stopped"); + if (throw_if_error) + throw Exception(ErrorCodes::LOGICAL_ERROR, "waitOtherHostsFinish() can't wait for other hosts to finish after the watching thread stopped"); + result = false; + return true; /// stop waiting + } + + LOG_TRACE(log, "Waiting for {} to finish{}{}", getHostDesc(host), reason_text, host_status); + return false; /// wait for next change of `state_changed` } LOG_TRACE(log, "Other hosts finished working on this {}", operation_name); + result = true; + return true; /// stop waiting +} + + +bool BackupCoordinationStageSync::finished() const +{ + std::lock_guard lock{mutex}; + return finishedNoLock(); +} + + +bool BackupCoordinationStageSync::finishedNoLock() const +{ + return state.hosts.at(current_host).finished; +} + + +bool BackupCoordinationStageSync::otherHostsFinished() const +{ + std::lock_guard lock{mutex}; + return otherHostsFinishedNoLock(); +} + + +bool BackupCoordinationStageSync::otherHostsFinishedNoLock() const +{ + for (const auto & [host, host_info] : state.hosts) + { + if (!host_info.finished && (host != current_host)) + return false; + } return true; } + +bool BackupCoordinationStageSync::allHostsFinishedNoLock() const +{ + return finishedNoLock() && otherHostsFinishedNoLock(); +} + + +Strings BackupCoordinationStageSync::getUnfinishedHosts() const +{ + std::lock_guard lock{mutex}; + return getUnfinishedHostsNoLock(); +} + + +Strings BackupCoordinationStageSync::getUnfinishedHostsNoLock() const +{ + if (allHostsFinishedNoLock()) + return {}; + + Strings res; + res.reserve(all_hosts.size()); + for (const auto & [host, host_info] : state.hosts) + { + if (!host_info.finished) + res.emplace_back(host); + } + return res; +} + + +Strings BackupCoordinationStageSync::getUnfinishedOtherHosts() const +{ + std::lock_guard lock{mutex}; + return getUnfinishedOtherHostsNoLock(); +} + + +Strings BackupCoordinationStageSync::getUnfinishedOtherHostsNoLock() const +{ + if (otherHostsFinishedNoLock()) + return {}; + + Strings res; + res.reserve(all_hosts.size() - 1); + for (const auto & [host, host_info] : state.hosts) + { + if (!host_info.finished && (host != current_host)) + res.emplace_back(host); + } + return res; +} + + +bool BackupCoordinationStageSync::setError(std::exception_ptr exception, bool throw_if_error) +{ + try + { + std::rethrow_exception(exception); + } + catch (const Exception & e) + { + return setError(e, throw_if_error); + } + catch (...) + { + return setError(Exception{getCurrentExceptionMessageAndPattern(true, true), getCurrentExceptionCode()}, throw_if_error); + } +} + + +bool BackupCoordinationStageSync::setError(const Exception & exception, bool throw_if_error) +{ + try + { + /// Most likely this exception has been already logged so here we're logging it without stacktrace. + String exception_message = getExceptionMessage(exception, /* with_stacktrace= */ false, /* check_embedded_stacktrace= */ true); + LOG_INFO(log, "Sending exception from {} to other hosts: {}", current_host_desc, exception_message); + + { + std::lock_guard lock{mutex}; + if (state.host_with_error) + { + LOG_INFO(log, "The error node already exists"); + return true; + } + + if (tried_to_set_error) + { + LOG_INFO(log, "Skipped creating the error node because earlier we failed to do that"); + return false; + } + } + + auto holder = with_retries.createRetriesControlHolder("BackupStageSync::setError", WithRetries::kErrorHandling); + holder.retries_ctl.retryLoop([&, &zookeeper = holder.faulty_zookeeper]() + { + with_retries.renewZooKeeper(zookeeper); + createErrorNode(exception, zookeeper); + }); + + { + std::lock_guard lock{mutex}; + tried_to_set_error = true; + return true; + } + } + catch (...) + { + LOG_TRACE(log, "Caught exception while removing nodes from ZooKeeper for this {}: {}", + is_restore ? "restore" : "backup", + getCurrentExceptionMessage(/* with_stacktrace= */ false, /* check_embedded_stacktrace= */ true)); + + std::lock_guard lock{mutex}; + tried_to_set_error = true; + + if (throw_if_error) + throw; + return false; + } +} + + +void BackupCoordinationStageSync::createErrorNode(const Exception & exception, Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper) +{ + String serialized_error; + { + WriteBufferFromOwnString buf; + writeStringBinary(current_host, buf); + writeException(exception, buf, true); + serialized_error = buf.str(); + } + + auto code = zookeeper->tryCreate(error_node_path, serialized_error, zkutil::CreateMode::Persistent); + + if (code == Coordination::Error::ZOK) + { + std::lock_guard lock{mutex}; + if (!state.host_with_error) + { + state.host_with_error = current_host; + state.hosts.at(current_host).exception = parseErrorNode(serialized_error).first; + } + LOG_TRACE(log, "Sent exception from {} to other hosts", current_host_desc); + return; + } + + if (code == Coordination::Error::ZNODEEXISTS) + { + String another_error = zookeeper->get(error_node_path); + auto [another_exception, host] = parseErrorNode(another_error); + if (another_exception) + { + std::lock_guard lock{mutex}; + if (!state.host_with_error) + { + state.host_with_error = host; + state.hosts.at(host).exception = another_exception; + } + LOG_INFO(log, "Another error is already assigned for this {}", operation_name); + return; + } + } + + throw zkutil::KeeperException::fromPath(code, error_node_path); +} + + +std::pair BackupCoordinationStageSync::parseErrorNode(const String & error_node_contents) const +{ + ReadBufferFromOwnString buf{error_node_contents}; + String host; + readStringBinary(host, buf); + if (std::find(all_hosts.begin(), all_hosts.end(), host) == all_hosts.end()) + return {}; + auto exception = std::make_exception_ptr(readException(buf, fmt::format("Got error from {}", getHostDesc(host)))); + return {exception, host}; +} + + +bool BackupCoordinationStageSync::isErrorSet() const +{ + std::lock_guard lock{mutex}; + return state.host_with_error.has_value(); +} + } diff --git a/src/Backups/BackupCoordinationStageSync.h b/src/Backups/BackupCoordinationStageSync.h index dc0d3c3c83d..11d3d1cf6f4 100644 --- a/src/Backups/BackupCoordinationStageSync.h +++ b/src/Backups/BackupCoordinationStageSync.h @@ -1,7 +1,9 @@ #pragma once +#include #include + namespace DB { @@ -9,12 +11,16 @@ namespace DB class BackupCoordinationStageSync { public: + /// Empty string as the current host is used to mark the initiator of a BACKUP ON CLUSTER or RESTORE ON CLUSTER query. + static const constexpr std::string_view kInitiator; + BackupCoordinationStageSync( bool is_restore_, /// true if this is a RESTORE ON CLUSTER command, false if this is a BACKUP ON CLUSTER command const String & zookeeper_path_, /// path to the "stage" folder in ZooKeeper const String & current_host_, /// the current host, or an empty string if it's the initiator of the BACKUP/RESTORE ON CLUSTER command const Strings & all_hosts_, /// all the hosts (including the initiator and the current host) performing the BACKUP/RESTORE ON CLUSTER command bool allow_concurrency_, /// whether it's allowed to have concurrent backups or restores. + BackupConcurrencyCounters & concurrency_counters_, const WithRetries & with_retries_, ThreadPoolCallbackRunnerUnsafe schedule_, QueryStatusPtr process_list_element_, @@ -22,30 +28,37 @@ public: ~BackupCoordinationStageSync(); + /// Sets that the BACKUP or RESTORE query was sent to other hosts. + void setQueryIsSentToOtherHosts(); + bool isQuerySentToOtherHosts() const; + /// Sets the stage of the current host and signal other hosts if there were other hosts waiting for that. void setStage(const String & stage, const String & stage_result = {}); - /// Waits until all the specified hosts come to the specified stage. - /// The function returns the results which specified hosts set when they came to the required stage. - /// If it doesn't happen before the timeout then the function will stop waiting and throw an exception. - Strings waitForHostsToReachStage(const String & stage_to_wait, const Strings & hosts, std::optional timeout = {}) const; - - /// Waits until all the other hosts finish their work. - /// Stops waiting and throws an exception if another host encounters an error or if some host gets cancelled. - void waitForOtherHostsToFinish() const; - - /// Lets other host know that the current host has finished its work. - void finish(bool & other_hosts_also_finished); + /// Waits until specified hosts come to the specified stage. + /// The function returns the results which the specified hosts set when they came to the required stage. + Strings waitHostsReachStage(const Strings & hosts, const String & stage_to_wait) const; /// Lets other hosts know that the current host has encountered an error. - bool trySetError(std::exception_ptr exception) noexcept; + /// The function returns true if it successfully created the error node or if the error node was found already exist. + bool setError(std::exception_ptr exception, bool throw_if_error); + bool isErrorSet() const; - /// Waits until all the other hosts finish their work (as a part of error-handling process). - /// Doesn't stops waiting if some host encounters an error or gets cancelled. - bool tryWaitForOtherHostsToFinishAfterError() const noexcept; + /// Waits until the hosts other than the current host finish their work. Must be called before finish(). + /// Stops waiting and throws an exception if another host encounters an error or if some host gets cancelled. + bool waitOtherHostsFinish(bool throw_if_error) const; + bool otherHostsFinished() const; - /// Lets other host know that the current host has finished its work (as a part of error-handling process). - bool tryFinishAfterError(bool & other_hosts_also_finished) noexcept; + /// Lets other hosts know that the current host has finished its work. + bool finish(bool throw_if_error); + bool finished() const; + + /// Returns true if all the hosts have finished. + bool allHostsFinished() const { return finished() && otherHostsFinished(); } + + /// Returns a list of the hosts which haven't finished yet. + Strings getUnfinishedHosts() const; + Strings getUnfinishedOtherHosts() const; /// Returns a printable name of a specific host. For empty host the function returns "initiator". static String getHostDesc(const String & host); @@ -78,14 +91,17 @@ private: /// Reads the current state from ZooKeeper without throwing exceptions. void readCurrentState(Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper); + + /// Creates a stage node to let other hosts know we've reached the specified stage. + void createStageNode(const String & stage, const String & stage_result, Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper); String getStageNodePath(const String & stage) const; /// Lets other hosts know that the current host has encountered an error. - bool trySetError(const Exception & exception); - void setError(const Exception & exception); + bool setError(const Exception & exception, bool throw_if_error); + void createErrorNode(const Exception & exception, Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper); /// Deserializes an error stored in the error node. - static std::pair parseErrorNode(const String & error_node_contents); + std::pair parseErrorNode(const String & error_node_contents) const; /// Reset the `connected` flag for each host. void resetConnectedFlag(); @@ -102,19 +118,27 @@ private: void cancelQueryIfDisconnectedTooLong(); /// Used by waitForHostsToReachStage() to check if everything is ready to return. - bool checkIfHostsReachStage(const Strings & hosts, const String & stage_to_wait, bool time_is_out, std::optional timeout, Strings & results) const TSA_REQUIRES(mutex); + bool checkIfHostsReachStage(const Strings & hosts, const String & stage_to_wait, Strings & results) const TSA_REQUIRES(mutex); /// Creates the 'finish' node. - bool tryFinishImpl(); - bool tryFinishImpl(bool & other_hosts_also_finished, bool throw_if_error, WithRetries::Kind retries_kind); - void createFinishNodeAndRemoveAliveNode(Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper); + bool finishImpl(bool throw_if_error, WithRetries::Kind retries_kind); + void createFinishNodeAndRemoveAliveNode(Coordination::ZooKeeperWithFaultInjection::Ptr zookeeper, bool throw_if_error); /// Returns the version used by the initiator. int getInitiatorVersion() const; /// Waits until all the other hosts finish their work. - bool tryWaitForOtherHostsToFinishImpl(const String & reason, bool throw_if_error, std::optional timeout) const; - bool checkIfOtherHostsFinish(const String & reason, bool throw_if_error, bool time_is_out, std::optional timeout) const TSA_REQUIRES(mutex); + bool waitOtherHostsFinishImpl(const String & reason, std::optional timeout, bool throw_if_error) const; + bool checkIfOtherHostsFinish(const String & reason, std::optional timeout, bool time_is_out, bool & result, bool throw_if_error) const TSA_REQUIRES(mutex); + + /// Returns true if all the hosts have finished. + bool allHostsFinishedNoLock() const TSA_REQUIRES(mutex); + bool finishedNoLock() const TSA_REQUIRES(mutex); + bool otherHostsFinishedNoLock() const TSA_REQUIRES(mutex); + + /// Returns a list of the hosts which haven't finished yet. + Strings getUnfinishedHostsNoLock() const TSA_REQUIRES(mutex); + Strings getUnfinishedOtherHostsNoLock() const TSA_REQUIRES(mutex); const bool is_restore; const String operation_name; @@ -138,15 +162,16 @@ private: /// Paths in ZooKeeper. const std::filesystem::path zookeeper_path; const String root_zookeeper_path; - const String operation_node_path; + const String operation_zookeeper_path; const String operation_node_name; - const String stage_node_path; const String start_node_path; const String finish_node_path; const String num_hosts_node_path; + const String error_node_path; const String alive_node_path; const String alive_tracker_node_path; - const String error_node_path; + + std::optional concurrency_check; std::shared_ptr zk_nodes_changed; @@ -176,25 +201,21 @@ private: { std::map hosts; /// std::map because we need to compare states std::optional host_with_error; - bool cancelled = false; bool operator ==(const State & other) const; bool operator !=(const State & other) const; + void merge(const State & other); }; State state TSA_GUARDED_BY(mutex); mutable std::condition_variable state_changed; std::future watching_thread_future; - std::atomic should_stop_watching_thread = false; + bool should_stop_watching_thread TSA_GUARDED_BY(mutex) = false; - struct FinishResult - { - bool succeeded = false; - std::exception_ptr exception; - bool other_hosts_also_finished = false; - }; - FinishResult finish_result TSA_GUARDED_BY(mutex); + bool query_is_sent_to_other_hosts TSA_GUARDED_BY(mutex) = false; + bool tried_to_finish TSA_GUARDED_BY(mutex) = false; + bool tried_to_set_error TSA_GUARDED_BY(mutex) = false; mutable std::mutex mutex; }; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 8480dc5d64d..88ebf8eef32 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -329,6 +329,7 @@ std::pair BackupsWorker::start(const ASTPtr & backup_ struct BackupsWorker::BackupStarter { BackupsWorker & backups_worker; + LoggerPtr log; std::shared_ptr backup_query; ContextPtr query_context; /// We have to keep `query_context` until the end of the operation because a pointer to it is stored inside the ThreadGroup we're using. ContextMutablePtr backup_context; @@ -345,6 +346,7 @@ struct BackupsWorker::BackupStarter BackupStarter(BackupsWorker & backups_worker_, const ASTPtr & query_, const ContextPtr & context_) : backups_worker(backups_worker_) + , log(backups_worker.log) , backup_query(std::static_pointer_cast(query_->clone())) , query_context(context_) , backup_context(Context::createCopy(query_context)) @@ -399,9 +401,20 @@ struct BackupsWorker::BackupStarter chassert(!backup); backup = backups_worker.openBackupForWriting(backup_info, backup_settings, backup_coordination, backup_context); - backups_worker.doBackup( - backup, backup_query, backup_id, backup_name_for_logging, backup_settings, backup_coordination, backup_context, - on_cluster, cluster); + backups_worker.doBackup(backup, backup_query, backup_id, backup_settings, backup_coordination, backup_context, + on_cluster, cluster); + + backup_coordination->finish(/* throw_if_error = */ true); + backup.reset(); + + /// The backup coordination is not needed anymore. + if (!is_internal_backup) + backup_coordination->cleanup(/* throw_if_error = */ true); + backup_coordination.reset(); + + /// NOTE: setStatus is called after setNumFilesAndSize in order to have actual information in a backup log record + LOG_INFO(log, "{} {} was created successfully", (is_internal_backup ? "Internal backup" : "Backup"), backup_name_for_logging); + backups_worker.setStatus(backup_id, BackupStatus::BACKUP_CREATED); } void onException() @@ -416,16 +429,29 @@ struct BackupsWorker::BackupStarter if (backup && !backup->setIsCorrupted()) should_remove_files_in_backup = false; - if (backup_coordination && backup_coordination->trySetError(std::current_exception())) + bool all_hosts_finished = false; + + if (backup_coordination && backup_coordination->setError(std::current_exception(), /* throw_if_error = */ false)) { - bool other_hosts_finished = backup_coordination->tryWaitForOtherHostsToFinishAfterError(); + bool other_hosts_finished = !is_internal_backup + && (!backup_coordination->isBackupQuerySentToOtherHosts() || backup_coordination->waitOtherHostsFinish(/* throw_if_error = */ false)); - if (should_remove_files_in_backup && other_hosts_finished) - backup->tryRemoveAllFiles(); - - backup_coordination->tryFinishAfterError(); + all_hosts_finished = backup_coordination->finish(/* throw_if_error = */ false) && other_hosts_finished; } + if (!all_hosts_finished) + should_remove_files_in_backup = false; + + if (backup && should_remove_files_in_backup) + backup->tryRemoveAllFiles(); + + backup.reset(); + + if (backup_coordination && all_hosts_finished) + backup_coordination->cleanup(/* throw_if_error = */ false); + + backup_coordination.reset(); + backups_worker.setStatusSafe(backup_id, getBackupStatusFromCurrentException()); } }; @@ -497,7 +523,6 @@ void BackupsWorker::doBackup( BackupMutablePtr backup, const std::shared_ptr & backup_query, const OperationID & backup_id, - const String & backup_name_for_logging, const BackupSettings & backup_settings, std::shared_ptr backup_coordination, ContextMutablePtr context, @@ -521,10 +546,10 @@ void BackupsWorker::doBackup( backup_settings.copySettingsToQuery(*backup_query); sendQueryToOtherHosts(*backup_query, cluster, backup_settings.shard_num, backup_settings.replica_num, context, required_access, backup_coordination->getOnClusterInitializationKeeperRetriesInfo()); - backup_coordination->setBackupQueryWasSentToOtherHosts(); + backup_coordination->setBackupQueryIsSentToOtherHosts(); /// Wait until all the hosts have written their backup entries. - backup_coordination->waitForOtherHostsToFinish(); + backup_coordination->waitOtherHostsFinish(/* throw_if_error = */ true); } else { @@ -569,18 +594,8 @@ void BackupsWorker::doBackup( compressed_size = backup->getCompressedSize(); } - /// Close the backup. - backup.reset(); - - /// The backup coordination is not needed anymore. - backup_coordination->finish(); - /// NOTE: we need to update metadata again after backup->finalizeWriting(), because backup metadata is written there. setNumFilesAndSize(backup_id, num_files, total_size, num_entries, uncompressed_size, compressed_size, 0, 0); - - /// NOTE: setStatus is called after setNumFilesAndSize in order to have actual information in a backup log record - LOG_INFO(log, "{} {} was created successfully", (is_internal_backup ? "Internal backup" : "Backup"), backup_name_for_logging); - setStatus(backup_id, BackupStatus::BACKUP_CREATED); } @@ -687,6 +702,7 @@ void BackupsWorker::writeBackupEntries( struct BackupsWorker::RestoreStarter { BackupsWorker & backups_worker; + LoggerPtr log; std::shared_ptr restore_query; ContextPtr query_context; /// We have to keep `query_context` until the end of the operation because a pointer to it is stored inside the ThreadGroup we're using. ContextMutablePtr restore_context; @@ -702,6 +718,7 @@ struct BackupsWorker::RestoreStarter RestoreStarter(BackupsWorker & backups_worker_, const ASTPtr & query_, const ContextPtr & context_) : backups_worker(backups_worker_) + , log(backups_worker.log) , restore_query(std::static_pointer_cast(query_->clone())) , query_context(context_) , restore_context(Context::createCopy(query_context)) @@ -753,16 +770,17 @@ struct BackupsWorker::RestoreStarter } restore_coordination = backups_worker.makeRestoreCoordination(on_cluster, restore_settings, restore_context); - backups_worker.doRestore( - restore_query, - restore_id, - backup_name_for_logging, - backup_info, - restore_settings, - restore_coordination, - restore_context, - on_cluster, - cluster); + backups_worker.doRestore(restore_query, restore_id, backup_info, restore_settings, restore_coordination, restore_context, + on_cluster, cluster); + + /// The restore coordination is not needed anymore. + restore_coordination->finish(/* throw_if_error = */ true); + if (!is_internal_restore) + restore_coordination->cleanup(/* throw_if_error = */ true); + restore_coordination.reset(); + + LOG_INFO(log, "Restored from {} {} successfully", (is_internal_restore ? "internal backup" : "backup"), backup_name_for_logging); + backups_worker.setStatus(restore_id, BackupStatus::RESTORED); } void onException() @@ -770,12 +788,16 @@ struct BackupsWorker::RestoreStarter /// Something bad happened, some data were not restored. tryLogCurrentException(backups_worker.log, fmt::format("Failed to restore from {} {}", (is_internal_restore ? "internal backup" : "backup"), backup_name_for_logging)); - if (restore_coordination && restore_coordination->trySetError(std::current_exception())) + if (restore_coordination && restore_coordination->setError(std::current_exception(), /* throw_if_error = */ false)) { - restore_coordination->tryWaitForOtherHostsToFinishAfterError(); - restore_coordination->tryFinishAfterError(); + bool other_hosts_finished = !is_internal_restore + && (!restore_coordination->isRestoreQuerySentToOtherHosts() || restore_coordination->waitOtherHostsFinish(/* throw_if_error = */ false)); + if (restore_coordination->finish(/* throw_if_error = */ false) && other_hosts_finished) + restore_coordination->cleanup(/* throw_if_error = */ false); } + restore_coordination.reset(); + backups_worker.setStatusSafe(restore_id, getRestoreStatusFromCurrentException()); } }; @@ -838,7 +860,6 @@ BackupPtr BackupsWorker::openBackupForReading(const BackupInfo & backup_info, co void BackupsWorker::doRestore( const std::shared_ptr & restore_query, const OperationID & restore_id, - const String & backup_name_for_logging, const BackupInfo & backup_info, RestoreSettings restore_settings, std::shared_ptr restore_coordination, @@ -882,10 +903,10 @@ void BackupsWorker::doRestore( restore_settings.copySettingsToQuery(*restore_query); sendQueryToOtherHosts(*restore_query, cluster, restore_settings.shard_num, restore_settings.replica_num, context, {}, restore_coordination->getOnClusterInitializationKeeperRetriesInfo()); - restore_coordination->setRestoreQueryWasSentToOtherHosts(); + restore_coordination->setRestoreQueryIsSentToOtherHosts(); /// Wait until all the hosts have done with their restoring work. - restore_coordination->waitForOtherHostsToFinish(); + restore_coordination->waitOtherHostsFinish(/* throw_if_error = */ true); } else { @@ -905,12 +926,6 @@ void BackupsWorker::doRestore( backup, context, getThreadPool(ThreadPoolId::RESTORE), after_task_callback}; restorer.run(RestorerFromBackup::RESTORE); } - - /// The restore coordination is not needed anymore. - restore_coordination->finish(); - - LOG_INFO(log, "Restored from {} {} successfully", (is_internal_restore ? "internal backup" : "backup"), backup_name_for_logging); - setStatus(restore_id, BackupStatus::RESTORED); } @@ -943,7 +958,7 @@ BackupsWorker::makeBackupCoordination(bool on_cluster, const BackupSettings & ba if (!on_cluster) { return std::make_shared( - *backup_settings.backup_uuid, !backup_settings.deduplicate_files, allow_concurrent_backups, *concurrency_counters); + !backup_settings.deduplicate_files, allow_concurrent_backups, *concurrency_counters); } bool is_internal_backup = backup_settings.internal; @@ -981,8 +996,7 @@ BackupsWorker::makeRestoreCoordination(bool on_cluster, const RestoreSettings & { if (!on_cluster) { - return std::make_shared( - *restore_settings.restore_uuid, allow_concurrent_restores, *concurrency_counters); + return std::make_shared(allow_concurrent_restores, *concurrency_counters); } bool is_internal_restore = restore_settings.internal; diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index 37f91e269a9..2e5ca84f3f6 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -81,7 +81,6 @@ private: BackupMutablePtr backup, const std::shared_ptr & backup_query, const BackupOperationID & backup_id, - const String & backup_name_for_logging, const BackupSettings & backup_settings, std::shared_ptr backup_coordination, ContextMutablePtr context, @@ -102,7 +101,6 @@ private: void doRestore( const std::shared_ptr & restore_query, const BackupOperationID & restore_id, - const String & backup_name_for_logging, const BackupInfo & backup_info, RestoreSettings restore_settings, std::shared_ptr restore_coordination, diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index c0eb90de89b..8bd874b9d0d 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -20,29 +20,27 @@ class IBackupCoordination public: virtual ~IBackupCoordination() = default; + /// Sets that the backup query was sent to other hosts. + /// Function waitOtherHostsFinish() will check that to find out if it should really wait or not. + virtual void setBackupQueryIsSentToOtherHosts() = 0; + virtual bool isBackupQuerySentToOtherHosts() const = 0; + /// Sets the current stage and waits for other hosts to come to this stage too. virtual Strings setStage(const String & new_stage, const String & message, bool sync) = 0; - /// Sets that the backup query was sent to other hosts. - /// Function waitForOtherHostsToFinish() will check that to find out if it should really wait or not. - virtual void setBackupQueryWasSentToOtherHosts() = 0; - /// Lets other hosts know that the current host has encountered an error. - virtual bool trySetError(std::exception_ptr exception) = 0; - - /// Lets other hosts know that the current host has finished its work. - virtual void finish() = 0; - - /// Lets other hosts know that the current host has finished its work (as a part of error-handling process). - virtual bool tryFinishAfterError() noexcept = 0; + /// Returns true if the information is successfully passed so other hosts can read it. + virtual bool setError(std::exception_ptr exception, bool throw_if_error) = 0; /// Waits until all the other hosts finish their work. /// Stops waiting and throws an exception if another host encounters an error or if some host gets cancelled. - virtual void waitForOtherHostsToFinish() = 0; + virtual bool waitOtherHostsFinish(bool throw_if_error) const = 0; - /// Waits until all the other hosts finish their work (as a part of error-handling process). - /// Doesn't stops waiting if some host encounters an error or gets cancelled. - virtual bool tryWaitForOtherHostsToFinishAfterError() noexcept = 0; + /// Lets other hosts know that the current host has finished its work. + virtual bool finish(bool throw_if_error) = 0; + + /// Removes temporary nodes in ZooKeeper. + virtual bool cleanup(bool throw_if_error) = 0; struct PartNameAndChecksum { diff --git a/src/Backups/IRestoreCoordination.h b/src/Backups/IRestoreCoordination.h index daabf1745f3..cc7bfd24202 100644 --- a/src/Backups/IRestoreCoordination.h +++ b/src/Backups/IRestoreCoordination.h @@ -18,29 +18,27 @@ class IRestoreCoordination public: virtual ~IRestoreCoordination() = default; + /// Sets that the restore query was sent to other hosts. + /// Function waitOtherHostsFinish() will check that to find out if it should really wait or not. + virtual void setRestoreQueryIsSentToOtherHosts() = 0; + virtual bool isRestoreQuerySentToOtherHosts() const = 0; + /// Sets the current stage and waits for other hosts to come to this stage too. virtual Strings setStage(const String & new_stage, const String & message, bool sync) = 0; - /// Sets that the restore query was sent to other hosts. - /// Function waitForOtherHostsToFinish() will check that to find out if it should really wait or not. - virtual void setRestoreQueryWasSentToOtherHosts() = 0; - /// Lets other hosts know that the current host has encountered an error. - virtual bool trySetError(std::exception_ptr exception) = 0; - - /// Lets other hosts know that the current host has finished its work. - virtual void finish() = 0; - - /// Lets other hosts know that the current host has finished its work (as a part of error-handling process). - virtual bool tryFinishAfterError() noexcept = 0; + /// Returns true if the information is successfully passed so other hosts can read it. + virtual bool setError(std::exception_ptr exception, bool throw_if_error) = 0; /// Waits until all the other hosts finish their work. /// Stops waiting and throws an exception if another host encounters an error or if some host gets cancelled. - virtual void waitForOtherHostsToFinish() = 0; + virtual bool waitOtherHostsFinish(bool throw_if_error) const = 0; - /// Waits until all the other hosts finish their work (as a part of error-handling process). - /// Doesn't stops waiting if some host encounters an error or gets cancelled. - virtual bool tryWaitForOtherHostsToFinishAfterError() noexcept = 0; + /// Lets other hosts know that the current host has finished its work. + virtual bool finish(bool throw_if_error) = 0; + + /// Removes temporary nodes in ZooKeeper. + virtual bool cleanup(bool throw_if_error) = 0; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. virtual bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) = 0; diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp index 569f58f1909..a9eee1fb159 100644 --- a/src/Backups/RestoreCoordinationLocal.cpp +++ b/src/Backups/RestoreCoordinationLocal.cpp @@ -10,9 +10,9 @@ namespace DB { RestoreCoordinationLocal::RestoreCoordinationLocal( - const UUID & restore_uuid, bool allow_concurrent_restore_, BackupConcurrencyCounters & concurrency_counters_) + bool allow_concurrent_restore_, BackupConcurrencyCounters & concurrency_counters_) : log(getLogger("RestoreCoordinationLocal")) - , concurrency_check(restore_uuid, /* is_restore = */ true, /* on_cluster = */ false, allow_concurrent_restore_, concurrency_counters_) + , concurrency_check(/* is_restore = */ true, /* on_cluster = */ false, /* zookeeper_path = */ "", allow_concurrent_restore_, concurrency_counters_) { } diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h index 6be357c4b7e..6e3262a8a2e 100644 --- a/src/Backups/RestoreCoordinationLocal.h +++ b/src/Backups/RestoreCoordinationLocal.h @@ -17,16 +17,16 @@ class ASTCreateQuery; class RestoreCoordinationLocal : public IRestoreCoordination { public: - RestoreCoordinationLocal(const UUID & restore_uuid_, bool allow_concurrent_restore_, BackupConcurrencyCounters & concurrency_counters_); + RestoreCoordinationLocal(bool allow_concurrent_restore_, BackupConcurrencyCounters & concurrency_counters_); ~RestoreCoordinationLocal() override; + void setRestoreQueryIsSentToOtherHosts() override {} + bool isRestoreQuerySentToOtherHosts() const override { return false; } Strings setStage(const String &, const String &, bool) override { return {}; } - void setRestoreQueryWasSentToOtherHosts() override {} - bool trySetError(std::exception_ptr) override { return true; } - void finish() override {} - bool tryFinishAfterError() noexcept override { return true; } - void waitForOtherHostsToFinish() override {} - bool tryWaitForOtherHostsToFinishAfterError() noexcept override { return true; } + bool setError(std::exception_ptr, bool) override { return true; } + bool waitOtherHostsFinish(bool) const override { return true; } + bool finish(bool) override { return true; } + bool cleanup(bool) override { return true; } /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; diff --git a/src/Backups/RestoreCoordinationOnCluster.cpp b/src/Backups/RestoreCoordinationOnCluster.cpp index 2029ad8b072..fad7341c044 100644 --- a/src/Backups/RestoreCoordinationOnCluster.cpp +++ b/src/Backups/RestoreCoordinationOnCluster.cpp @@ -35,17 +35,21 @@ RestoreCoordinationOnCluster::RestoreCoordinationOnCluster( , current_host_index(BackupCoordinationOnCluster::findCurrentHostIndex(current_host, all_hosts)) , log(getLogger("RestoreCoordinationOnCluster")) , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); }) - , concurrency_check(restore_uuid_, /* is_restore = */ true, /* on_cluster = */ true, allow_concurrent_restore_, concurrency_counters_) - , stage_sync(/* is_restore = */ true, fs::path{zookeeper_path} / "stage", current_host, all_hosts, allow_concurrent_restore_, with_retries, schedule_, process_list_element_, log) - , cleaner(zookeeper_path, with_retries, log) + , cleaner(/* is_restore = */ true, zookeeper_path, with_retries, log) + , stage_sync(/* is_restore = */ true, fs::path{zookeeper_path} / "stage", current_host, all_hosts, allow_concurrent_restore_, concurrency_counters_, with_retries, schedule_, process_list_element_, log) { - createRootNodes(); + try + { + createRootNodes(); + } + catch (...) + { + stage_sync.setError(std::current_exception(), /* throw_if_error = */ false); + throw; + } } -RestoreCoordinationOnCluster::~RestoreCoordinationOnCluster() -{ - tryFinishImpl(); -} +RestoreCoordinationOnCluster::~RestoreCoordinationOnCluster() = default; void RestoreCoordinationOnCluster::createRootNodes() { @@ -66,69 +70,52 @@ void RestoreCoordinationOnCluster::createRootNodes() }); } +void RestoreCoordinationOnCluster::setRestoreQueryIsSentToOtherHosts() +{ + stage_sync.setQueryIsSentToOtherHosts(); +} + +bool RestoreCoordinationOnCluster::isRestoreQuerySentToOtherHosts() const +{ + return stage_sync.isQuerySentToOtherHosts(); +} + Strings RestoreCoordinationOnCluster::setStage(const String & new_stage, const String & message, bool sync) { stage_sync.setStage(new_stage, message); - - if (!sync) - return {}; - - return stage_sync.waitForHostsToReachStage(new_stage, all_hosts_without_initiator); + if (sync) + return stage_sync.waitHostsReachStage(all_hosts_without_initiator, new_stage); + return {}; } -void RestoreCoordinationOnCluster::setRestoreQueryWasSentToOtherHosts() +bool RestoreCoordinationOnCluster::setError(std::exception_ptr exception, bool throw_if_error) { - restore_query_was_sent_to_other_hosts = true; + return stage_sync.setError(exception, throw_if_error); } -bool RestoreCoordinationOnCluster::trySetError(std::exception_ptr exception) +bool RestoreCoordinationOnCluster::waitOtherHostsFinish(bool throw_if_error) const { - return stage_sync.trySetError(exception); + return stage_sync.waitOtherHostsFinish(throw_if_error); } -void RestoreCoordinationOnCluster::finish() +bool RestoreCoordinationOnCluster::finish(bool throw_if_error) { - bool other_hosts_also_finished = false; - stage_sync.finish(other_hosts_also_finished); - - if ((current_host == kInitiator) && (other_hosts_also_finished || !restore_query_was_sent_to_other_hosts)) - cleaner.cleanup(); + return stage_sync.finish(throw_if_error); } -bool RestoreCoordinationOnCluster::tryFinishAfterError() noexcept +bool RestoreCoordinationOnCluster::cleanup(bool throw_if_error) { - return tryFinishImpl(); -} - -bool RestoreCoordinationOnCluster::tryFinishImpl() noexcept -{ - bool other_hosts_also_finished = false; - if (!stage_sync.tryFinishAfterError(other_hosts_also_finished)) - return false; - - if ((current_host == kInitiator) && (other_hosts_also_finished || !restore_query_was_sent_to_other_hosts)) + /// All the hosts must finish before we remove the coordination nodes. + bool expect_other_hosts_finished = stage_sync.isQuerySentToOtherHosts() || !stage_sync.isErrorSet(); + bool all_hosts_finished = stage_sync.finished() && (stage_sync.otherHostsFinished() || !expect_other_hosts_finished); + if (!all_hosts_finished) { - if (!cleaner.tryCleanupAfterError()) - return false; - } - - return true; -} - -void RestoreCoordinationOnCluster::waitForOtherHostsToFinish() -{ - if ((current_host != kInitiator) || !restore_query_was_sent_to_other_hosts) - return; - stage_sync.waitForOtherHostsToFinish(); -} - -bool RestoreCoordinationOnCluster::tryWaitForOtherHostsToFinishAfterError() noexcept -{ - if (current_host != kInitiator) + auto unfinished_hosts = expect_other_hosts_finished ? stage_sync.getUnfinishedHosts() : Strings{current_host}; + LOG_INFO(log, "Skipping removing nodes from ZooKeeper because hosts {} didn't finish", + BackupCoordinationStageSync::getHostsDesc(unfinished_hosts)); return false; - if (!restore_query_was_sent_to_other_hosts) - return true; - return stage_sync.tryWaitForOtherHostsToFinishAfterError(); + } + return cleaner.cleanup(throw_if_error); } ZooKeeperRetriesInfo RestoreCoordinationOnCluster::getOnClusterInitializationKeeperRetriesInfo() const diff --git a/src/Backups/RestoreCoordinationOnCluster.h b/src/Backups/RestoreCoordinationOnCluster.h index 87a8dd3ce83..99929cbdac3 100644 --- a/src/Backups/RestoreCoordinationOnCluster.h +++ b/src/Backups/RestoreCoordinationOnCluster.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -15,7 +14,7 @@ class RestoreCoordinationOnCluster : public IRestoreCoordination { public: /// Empty string as the current host is used to mark the initiator of a RESTORE ON CLUSTER query. - static const constexpr std::string_view kInitiator; + static const constexpr std::string_view kInitiator = BackupCoordinationStageSync::kInitiator; RestoreCoordinationOnCluster( const UUID & restore_uuid_, @@ -31,13 +30,13 @@ public: ~RestoreCoordinationOnCluster() override; + void setRestoreQueryIsSentToOtherHosts() override; + bool isRestoreQuerySentToOtherHosts() const override; Strings setStage(const String & new_stage, const String & message, bool sync) override; - void setRestoreQueryWasSentToOtherHosts() override; - bool trySetError(std::exception_ptr exception) override; - void finish() override; - bool tryFinishAfterError() noexcept override; - void waitForOtherHostsToFinish() override; - bool tryWaitForOtherHostsToFinishAfterError() noexcept override; + bool setError(std::exception_ptr exception, bool throw_if_error) override; + bool waitOtherHostsFinish(bool throw_if_error) const override; + bool finish(bool throw_if_error) override; + bool cleanup(bool throw_if_error) override; /// Starts creating a table in a replicated database. Returns false if there is another host which is already creating this table. bool acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) override; @@ -78,11 +77,10 @@ private: const size_t current_host_index; LoggerPtr const log; + /// The order is important: `stage_sync` must be initialized after `with_retries` and `cleaner`. const WithRetries with_retries; - BackupConcurrencyCheck concurrency_check; - BackupCoordinationStageSync stage_sync; BackupCoordinationCleaner cleaner; - std::atomic restore_query_was_sent_to_other_hosts = false; + BackupCoordinationStageSync stage_sync; }; } diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index e8e009ec306..7b4651a638f 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -140,8 +140,6 @@ void highlight(const String & query, std::vector & colors /// We don't do highlighting for foreign dialects, such as PRQL and Kusto. /// Only normal ClickHouse SQL queries are highlighted. - /// Currently we highlight only the first query in the multi-query mode. - ParserQuery parser(end, false, context.getSettingsRef()[Setting::implicit_select]); ASTPtr ast; bool parse_res = false; diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index f419a23b209..262f3ca3a7a 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -331,7 +331,7 @@ AsynchronousMetrics::~AsynchronousMetrics() AsynchronousMetricValues AsynchronousMetrics::getValues() const { - std::lock_guard lock(data_mutex); + SharedLockGuard lock(values_mutex); return values; } @@ -1807,7 +1807,10 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) first_run = false; // Finally, update the current metrics. - values = new_values; + { + std::lock_guard values_lock(values_mutex); + values.swap(new_values); + } } } diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 215dc6e1337..e8d4b7980f9 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -100,6 +101,7 @@ private: std::condition_variable wait_cond; bool quit TSA_GUARDED_BY(thread_mutex) = false; + /// Protects all raw data and serializes multiple updates. mutable std::mutex data_mutex; /// Some values are incremental and we have to calculate the difference. @@ -107,7 +109,15 @@ private: bool first_run TSA_GUARDED_BY(data_mutex) = true; TimePoint previous_update_time TSA_GUARDED_BY(data_mutex); - AsynchronousMetricValues values TSA_GUARDED_BY(data_mutex); + /// Protects saved values. + mutable SharedMutex values_mutex; + /// Values store the result of the last update prepared for reading. +#ifdef OS_LINUX + AsynchronousMetricValues values TSA_GUARDED_BY(values_mutex); +#else + /// When SharedMutex == std::shared_mutex it may not be annotated with the 'capability'. + AsynchronousMetricValues values; +#endif #if defined(OS_LINUX) || defined(OS_FREEBSD) MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 0526596509d..0a987cf36af 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -255,6 +255,7 @@ M(PartsActive, "Active data part, used by current and upcoming SELECTs.") \ M(AttachedDatabase, "Active databases.") \ M(AttachedTable, "Active tables.") \ + M(AttachedReplicatedTable, "Active replicated tables.") \ M(AttachedView, "Active views.") \ M(AttachedDictionary, "Active dictionaries.") \ M(PartsOutdated, "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.") \ diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 0898bdded83..bc1b604d0e5 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -87,6 +87,7 @@ APPLY_FOR_FAILPOINTS(M, M, M, M) std::unordered_map> FailPointInjection::fail_point_wait_channels; std::mutex FailPointInjection::mu; + class FailPointChannel : private boost::noncopyable { public: diff --git a/src/Common/FailPoint.h b/src/Common/FailPoint.h index 1af13d08553..69aa8d75b5b 100644 --- a/src/Common/FailPoint.h +++ b/src/Common/FailPoint.h @@ -15,6 +15,7 @@ #include + namespace DB { @@ -27,6 +28,7 @@ namespace DB /// 3. in test file, we can use system failpoint enable/disable 'failpoint_name' class FailPointChannel; + class FailPointInjection { public: diff --git a/src/Common/HostResolvePool.cpp b/src/Common/HostResolvePool.cpp index e8a05a269bc..2c5bf039fe5 100644 --- a/src/Common/HostResolvePool.cpp +++ b/src/Common/HostResolvePool.cpp @@ -9,6 +9,7 @@ #include #include +#include namespace ProfileEvents @@ -49,16 +50,18 @@ HostResolver::WeakPtr HostResolver::getWeakFromThis() } HostResolver::HostResolver(String host_, Poco::Timespan history_) - : host(std::move(host_)) - , history(history_) - , resolve_function([](const String & host_to_resolve) { return DNSResolver::instance().resolveHostAllInOriginOrder(host_to_resolve); }) -{ - update(); -} + : HostResolver( + [](const String & host_to_resolve) { return DNSResolver::instance().resolveHostAllInOriginOrder(host_to_resolve); }, + host_, + history_) +{} HostResolver::HostResolver( ResolveFunction && resolve_function_, String host_, Poco::Timespan history_) - : host(std::move(host_)), history(history_), resolve_function(std::move(resolve_function_)) + : host(std::move(host_)) + , history(history_) + , resolve_interval(history_.totalMicroseconds() / 3) + , resolve_function(std::move(resolve_function_)) { update(); } @@ -203,7 +206,7 @@ bool HostResolver::isUpdateNeeded() Poco::Timestamp now; std::lock_guard lock(mutex); - return last_resolve_time + history < now || records.empty(); + return last_resolve_time + resolve_interval < now || records.empty(); } void HostResolver::updateImpl(Poco::Timestamp now, std::vector & next_gen) diff --git a/src/Common/HostResolvePool.h b/src/Common/HostResolvePool.h index d148e909ca9..b979da3d142 100644 --- a/src/Common/HostResolvePool.h +++ b/src/Common/HostResolvePool.h @@ -26,7 +26,7 @@ // a) it still occurs in resolve set after `history_` time or b) all other addresses are pessimized as well. // - resolve schedule // Addresses are resolved through `DB::DNSResolver::instance()`. -// Usually it does not happen more often than once in `history_` time. +// Usually it does not happen more often than 3 times in `history_` period. // But also new resolve performed each `setFail()` call. namespace DB @@ -212,6 +212,7 @@ protected: const String host; const Poco::Timespan history; + const Poco::Timespan resolve_interval; const HostResolverMetrics metrics = getMetrics(); // for tests purpose @@ -245,4 +246,3 @@ private: }; } - diff --git a/src/Common/LockGuard.h b/src/Common/LockGuard.h index 8a98c5f553a..03c8a3e7617 100644 --- a/src/Common/LockGuard.h +++ b/src/Common/LockGuard.h @@ -1,23 +1,47 @@ #pragma once -#include #include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +}; + /** LockGuard provides RAII-style locking mechanism for a mutex. - ** It's intended to be used like std::unique_ptr but with TSA annotations + ** It's intended to be used like std::unique_lock but with TSA annotations */ template class TSA_SCOPED_LOCKABLE LockGuard { public: - explicit LockGuard(Mutex & mutex_) TSA_ACQUIRE(mutex_) : mutex(mutex_) { mutex.lock(); } - ~LockGuard() TSA_RELEASE() { mutex.unlock(); } + explicit LockGuard(Mutex & mutex_) TSA_ACQUIRE(mutex_) : mutex(mutex_) { lock(); } + ~LockGuard() TSA_RELEASE() { if (locked) unlock(); } + + void lock() TSA_ACQUIRE() + { + /// Don't allow recursive_mutex for now. + if (locked) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't lock twice the same mutex"); + mutex.lock(); + locked = true; + } + + void unlock() TSA_RELEASE() + { + if (!locked) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't unlock the mutex without locking it first"); + mutex.unlock(); + locked = false; + } private: Mutex & mutex; + bool locked = false; }; template typename TLockGuard, typename Mutex> diff --git a/src/Compression/CompressionCodecNone.h b/src/Compression/CompressionCodecNone.h index 5d6f135b351..0aaa973b55f 100644 --- a/src/Compression/CompressionCodecNone.h +++ b/src/Compression/CompressionCodecNone.h @@ -3,7 +3,7 @@ #include #include #include -#include + namespace DB { diff --git a/src/Compression/getCompressionCodecForFile.cpp b/src/Compression/getCompressionCodecForFile.cpp index 027ee0ac57a..b04e4b6371a 100644 --- a/src/Compression/getCompressionCodecForFile.cpp +++ b/src/Compression/getCompressionCodecForFile.cpp @@ -10,33 +10,50 @@ namespace DB { - using Checksum = CityHash_v1_0_2::uint128; -CompressionCodecPtr getCompressionCodecForFile(const IDataPartStorage & data_part_storage, const String & relative_path) +CompressionCodecPtr +getCompressionCodecForFile(ReadBuffer & read_buffer, UInt32 & size_compressed, UInt32 & size_decompressed, bool skip_to_next_block) { - auto read_buffer = data_part_storage.readFile(relative_path, {}, std::nullopt, std::nullopt); - read_buffer->ignore(sizeof(Checksum)); + read_buffer.ignore(sizeof(Checksum)); UInt8 header_size = ICompressionCodec::getHeaderSize(); + size_t starting_bytes = read_buffer.count(); PODArray compressed_buffer; compressed_buffer.resize(header_size); - read_buffer->readStrict(compressed_buffer.data(), header_size); + read_buffer.readStrict(compressed_buffer.data(), header_size); uint8_t method = ICompressionCodec::readMethod(compressed_buffer.data()); + size_compressed = unalignedLoad(&compressed_buffer[1]); + size_decompressed = unalignedLoad(&compressed_buffer[5]); if (method == static_cast(CompressionMethodByte::Multiple)) { compressed_buffer.resize(1); - read_buffer->readStrict(compressed_buffer.data(), 1); + read_buffer.readStrict(compressed_buffer.data(), 1); compressed_buffer.resize(1 + compressed_buffer[0]); - read_buffer->readStrict(compressed_buffer.data() + 1, compressed_buffer[0]); + read_buffer.readStrict(compressed_buffer.data() + 1, compressed_buffer[0]); auto codecs_bytes = CompressionCodecMultiple::getCodecsBytesFromData(compressed_buffer.data()); Codecs codecs; for (auto byte : codecs_bytes) codecs.push_back(CompressionCodecFactory::instance().get(byte)); + if (skip_to_next_block) + read_buffer.ignore(size_compressed - (read_buffer.count() - starting_bytes)); + return std::make_shared(codecs); } + + if (skip_to_next_block) + read_buffer.ignore(size_compressed - (read_buffer.count() - starting_bytes)); + return CompressionCodecFactory::instance().get(method); } +CompressionCodecPtr getCompressionCodecForFile(const IDataPartStorage & data_part_storage, const String & relative_path) +{ + auto read_buffer = data_part_storage.readFile(relative_path, {}, std::nullopt, std::nullopt); + UInt32 size_compressed; + UInt32 size_decompressed; + return getCompressionCodecForFile(*read_buffer, size_compressed, size_decompressed, false); +} + } diff --git a/src/Compression/getCompressionCodecForFile.h b/src/Compression/getCompressionCodecForFile.h index b6f22750e4d..535befa37e1 100644 --- a/src/Compression/getCompressionCodecForFile.h +++ b/src/Compression/getCompressionCodecForFile.h @@ -13,4 +13,8 @@ namespace DB /// from metadata. CompressionCodecPtr getCompressionCodecForFile(const IDataPartStorage & data_part_storage, const String & relative_path); +/// Same as above which is used by clickhouse-compressor to print compression statistics of each data block. +CompressionCodecPtr +getCompressionCodecForFile(ReadBuffer & read_buffer, UInt32 & size_compressed, UInt32 & size_decompressed, bool skip_to_next_block); + } diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index 283ccd95f14..8265ba63fc2 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index d573377fc8b..2f8e7b6843a 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -131,6 +131,9 @@ namespace DB DECLARE(UInt64, max_database_num_to_warn, 1000lu, "If the number of databases is greater than this value, the server will create a warning that will displayed to user.", 0) \ DECLARE(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \ DECLARE(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ + DECLARE(UInt64, max_replicated_table_num_to_throw, 0lu, "If number of replicated tables is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ + DECLARE(UInt64, max_dictionary_num_to_throw, 0lu, "If number of dictionaries is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ + DECLARE(UInt64, max_view_num_to_throw, 0lu, "If number of views is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ DECLARE(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \ DECLARE(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users. Zero means unlimited", 0) \ DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \ diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 7c9e0b236ac..15861289648 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -3669,6 +3669,11 @@ Given that, for example, dictionaries, can be out of sync across nodes, mutation ``` +)", 0) \ + DECLARE(Bool, validate_mutation_query, true, R"( +Validate mutation queries before accepting them. Mutations are executed in the background, and running an invalid query will cause mutations to get stuck, requiring manual intervention. + +Only change this setting if you encounter a backward-incompatible bug. )", 0) \ DECLARE(Seconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, R"( Defines how many seconds a locking request waits before failing. @@ -4861,9 +4866,9 @@ Allows to record the filesystem caching log for each query DECLARE(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, R"( Allow to use the filesystem cache in passive mode - benefit from the existing cache entries, but don't put more entries into the cache. If you set this setting for heavy ad-hoc queries and leave it disabled for short real-time queries, this will allows to avoid cache threshing by too heavy queries and to improve the overall system efficiency. )", 0) \ - DECLARE(Bool, skip_download_if_exceeds_query_cache, true, R"( + DECLARE(Bool, filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit, true, R"( Skip download from remote filesystem if exceeds query cache size -)", 0) \ +)", 0) ALIAS(skip_download_if_exceeds_query_cache) \ DECLARE(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), R"( Max remote filesystem cache size that can be downloaded by a single query )", 0) \ @@ -4875,6 +4880,9 @@ Limit on size of a single batch of file segments that a read buffer can request )", 0) \ DECLARE(UInt64, filesystem_cache_reserve_space_wait_lock_timeout_milliseconds, 1000, R"( Wait time to lock cache for space reservation in filesystem cache +)", 0) \ + DECLARE(Bool, filesystem_cache_prefer_bigger_buffer_size, true, R"( +Prefer bigger buffer size if filesystem cache is enabled to avoid writing small file segments which deteriorate cache performance. On the other hand, enabling this setting might increase memory usage. )", 0) \ DECLARE(UInt64, temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds, (10 * 60 * 1000), R"( Wait time to lock cache for space reservation for temporary data in filesystem cache diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 588cbb0698b..9736fef9af3 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -65,6 +65,7 @@ static std::initializer_listisSystemStorage() && !DatabaseCatalog::isPredefinedDatabase(database_name)) { LOG_TEST(log, "Counting detached table {} to database {}", table_name, database_name); - CurrentMetrics::sub(getAttachedCounterForStorage(table_storage)); + for (auto metric : getAttachedCountersForStorage(table_storage)) + CurrentMetrics::sub(metric); } auto table_id = table_storage->getStorageID(); @@ -430,7 +431,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c if (!table->isSystemStorage() && !DatabaseCatalog::isPredefinedDatabase(database_name)) { LOG_TEST(log, "Counting attached table {} to database {}", table_name, database_name); - CurrentMetrics::add(getAttachedCounterForStorage(table)); + for (auto metric : getAttachedCountersForStorage(table)) + CurrentMetrics::add(metric); } } diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 673c82806bd..b8386bcf967 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -277,19 +277,6 @@ void AzureObjectStorage::removeObjectImpl(const StoredObject & object, const Sha } } -/// Remove file. Throws exception if file doesn't exists or it's a directory. -void AzureObjectStorage::removeObject(const StoredObject & object) -{ - removeObjectImpl(object, client.get(), false); -} - -void AzureObjectStorage::removeObjects(const StoredObjects & objects) -{ - auto client_ptr = client.get(); - for (const auto & object : objects) - removeObjectImpl(object, client_ptr, false); -} - void AzureObjectStorage::removeObjectIfExists(const StoredObject & object) { removeObjectImpl(object, client.get(), true); diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 58225eccd90..401493be367 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -59,11 +59,6 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - /// Remove file. Throws exception if file doesn't exists or it's a directory. - void removeObject(const StoredObject & object) override; - - void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const StoredObject & object) override; void removeObjectsIfExist(const StoredObjects & objects) override; diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp index 163ff3a9c68..779b8830fab 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp @@ -148,20 +148,6 @@ void CachedObjectStorage::removeCacheIfExists(const std::string & path_key_for_c cache->removeKeyIfExists(getCacheKey(path_key_for_cache), FileCache::getCommonUser().user_id); } -void CachedObjectStorage::removeObject(const StoredObject & object) -{ - removeCacheIfExists(object.remote_path); - object_storage->removeObject(object); -} - -void CachedObjectStorage::removeObjects(const StoredObjects & objects) -{ - for (const auto & object : objects) - removeCacheIfExists(object.remote_path); - - object_storage->removeObjects(objects); -} - void CachedObjectStorage::removeObjectIfExists(const StoredObject & object) { removeCacheIfExists(object.remote_path); diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h index b77baf21e40..77aa635b89b 100644 --- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h +++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h @@ -45,10 +45,6 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void removeObject(const StoredObject & object) override; - - void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const StoredObject & object) override; void removeObjectsIfExist(const StoredObjects & objects) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index 3720c04a471..fba45d5a0c9 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -642,7 +642,10 @@ std::unique_ptr DiskObjectStorage::readFile( }; /// Avoid cache fragmentation by choosing bigger buffer size. - bool prefer_bigger_buffer_size = object_storage->supportsCache() && read_settings.enable_filesystem_cache; + bool prefer_bigger_buffer_size = read_settings.filesystem_cache_prefer_bigger_buffer_size + && object_storage->supportsCache() + && read_settings.enable_filesystem_cache; + size_t buffer_size = prefer_bigger_buffer_size ? std::max(settings.remote_fs_buffer_size, DBMS_DEFAULT_BUFFER_SIZE) : settings.remote_fs_buffer_size; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index 64323fb6f3c..19de2bb78af 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -480,8 +480,7 @@ struct WriteFileObjectStorageOperation final : public IDiskObjectStorageOperatio void undo() override { - if (object_storage.exists(object)) - object_storage.removeObject(object); + object_storage.removeObjectIfExists(object); } void finalize() override @@ -543,8 +542,7 @@ struct CopyFileObjectStorageOperation final : public IDiskObjectStorageOperation void undo() override { - for (const auto & object : created_objects) - destination_object_storage.removeObject(object); + destination_object_storage.removeObjectsIfExist(created_objects); } void finalize() override diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h index b53161beb76..7d6c914c398 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h @@ -77,11 +77,6 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - /// Remove file. Throws exception if file doesn't exists or it's a directory. - void removeObject(const StoredObject & object) override; - - void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const StoredObject & object) override; void removeObjectsIfExist(const StoredObjects & objects) override; @@ -117,6 +112,11 @@ private: void initializeHDFSFS() const; std::string extractObjectKeyFromURL(const StoredObject & object) const; + /// Remove file. Throws exception if file doesn't exists or it's a directory. + void removeObject(const StoredObject & object); + + void removeObjects(const StoredObjects & objects); + const Poco::Util::AbstractConfiguration & config; mutable HDFSBuilderWrapper hdfs_builder; diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h index 8dde96b8b16..adb36762539 100644 --- a/src/Disks/ObjectStorages/IObjectStorage.h +++ b/src/Disks/ObjectStorages/IObjectStorage.h @@ -161,11 +161,11 @@ public: virtual bool isRemote() const = 0; /// Remove object. Throws exception if object doesn't exists. - virtual void removeObject(const StoredObject & object) = 0; + // virtual void removeObject(const StoredObject & object) = 0; /// Remove multiple objects. Some object storages can do batch remove in a more /// optimal way. - virtual void removeObjects(const StoredObjects & objects) = 0; + // virtual void removeObjects(const StoredObjects & objects) = 0; /// Remove object on path if exists virtual void removeObjectIfExists(const StoredObject & object) = 0; diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index 5f1b6aedc72..f24501dc60e 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -81,7 +81,7 @@ std::unique_ptr LocalObjectStorage::writeObject( /// NO return std::make_unique(object.remote_path, buf_size); } -void LocalObjectStorage::removeObject(const StoredObject & object) +void LocalObjectStorage::removeObject(const StoredObject & object) const { /// For local object storage files are actually removed when "metadata" is removed. if (!exists(object)) @@ -91,7 +91,7 @@ void LocalObjectStorage::removeObject(const StoredObject & object) ErrnoException::throwFromPath(ErrorCodes::CANNOT_UNLINK, object.remote_path, "Cannot unlink file {}", object.remote_path); } -void LocalObjectStorage::removeObjects(const StoredObjects & objects) +void LocalObjectStorage::removeObjects(const StoredObjects & objects) const { for (const auto & object : objects) removeObject(object); diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h index f1a0391a984..5b3c3951364 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h @@ -42,10 +42,6 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void removeObject(const StoredObject & object) override; - - void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const StoredObject & object) override; void removeObjectsIfExist(const StoredObjects & objects) override; @@ -82,6 +78,10 @@ public: ReadSettings patchSettings(const ReadSettings & read_settings) const override; private: + void removeObject(const StoredObject & object) const; + + void removeObjects(const StoredObjects & objects) const; + String key_prefix; LoggerPtr log; std::string description; diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp index d56c5d9143c..27aa9304de7 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp @@ -203,7 +203,7 @@ void MetadataStorageFromPlainObjectStorageTransaction::unlinkFile(const std::str { auto object_key = metadata_storage.object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */); auto object = StoredObject(object_key.serialize()); - metadata_storage.object_storage->removeObject(object); + metadata_storage.object_storage->removeObjectIfExists(object); } void MetadataStorageFromPlainObjectStorageTransaction::removeDirectory(const std::string & path) @@ -211,7 +211,7 @@ void MetadataStorageFromPlainObjectStorageTransaction::removeDirectory(const std if (metadata_storage.object_storage->isWriteOnce()) { for (auto it = metadata_storage.iterateDirectory(path); it->isValid(); it->next()) - metadata_storage.object_storage->removeObject(StoredObject(it->path())); + metadata_storage.object_storage->removeObjectIfExists(StoredObject(it->path())); } else { diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp index ea57d691908..62015631aa5 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp @@ -107,7 +107,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::un auto metric = object_storage->getMetadataStorageMetrics().directory_map_size; CurrentMetrics::sub(metric, 1); - object_storage->removeObject(StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME)); + object_storage->removeObjectIfExists(StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME)); } else if (write_created) object_storage->removeObjectIfExists(StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME)); @@ -247,7 +247,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std: auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix); auto metadata_object = StoredObject(/*remote_path*/ metadata_object_key.serialize(), /*local_path*/ path / PREFIX_PATH_FILE_NAME); - object_storage->removeObject(metadata_object); + object_storage->removeObjectIfExists(metadata_object); { std::lock_guard lock(path_map.mutex); diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 47ef97401f2..9fca3cad688 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -326,21 +326,11 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e ProfileEvents::DiskS3DeleteObjects); } -void S3ObjectStorage::removeObject(const StoredObject & object) -{ - removeObjectImpl(object, false); -} - void S3ObjectStorage::removeObjectIfExists(const StoredObject & object) { removeObjectImpl(object, true); } -void S3ObjectStorage::removeObjects(const StoredObjects & objects) -{ - removeObjectsImpl(objects, false); -} - void S3ObjectStorage::removeObjectsIfExist(const StoredObjects & objects) { removeObjectsImpl(objects, true); diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h index d6e84cf57ef..4b9c968ede9 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h @@ -101,13 +101,6 @@ public: ObjectStorageIteratorPtr iterate(const std::string & path_prefix, size_t max_keys) const override; - /// Uses `DeleteObjectRequest`. - void removeObject(const StoredObject & object) override; - - /// Uses `DeleteObjectsRequest` if it is allowed by `s3_capabilities`, otherwise `DeleteObjectRequest`. - /// `DeleteObjectsRequest` is not supported on GCS, see https://issuetracker.google.com/issues/162653700 . - void removeObjects(const StoredObjects & objects) override; - /// Uses `DeleteObjectRequest`. void removeObjectIfExists(const StoredObject & object) override; diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp index 871d3b506f6..35abc0ed0df 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp @@ -254,16 +254,6 @@ std::unique_ptr WebObjectStorage::writeObject( /// NOLI throwNotAllowed(); } -void WebObjectStorage::removeObject(const StoredObject &) -{ - throwNotAllowed(); -} - -void WebObjectStorage::removeObjects(const StoredObjects &) -{ - throwNotAllowed(); -} - void WebObjectStorage::removeObjectIfExists(const StoredObject &) { throwNotAllowed(); diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h index 573221b7e21..1e612bd359c 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h @@ -47,10 +47,6 @@ public: size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, const WriteSettings & write_settings = {}) override; - void removeObject(const StoredObject & object) override; - - void removeObjects(const StoredObjects & objects) override; - void removeObjectIfExists(const StoredObject & object) override; void removeObjectsIfExist(const StoredObjects & objects) override; diff --git a/src/Formats/EscapingRuleUtils.cpp b/src/Formats/EscapingRuleUtils.cpp index 67371784712..687814baaa6 100644 --- a/src/Formats/EscapingRuleUtils.cpp +++ b/src/Formats/EscapingRuleUtils.cpp @@ -10,7 +10,6 @@ #include #include #include -#include namespace DB diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index be0875581a5..9b2328065fc 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1033,6 +1033,9 @@ private: size_t tuple_size, size_t input_rows_count) const { + if (0 == tuple_size) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Comparison of zero-sized tuples is not implemented"); + ColumnsWithTypeAndName less_columns(tuple_size); ColumnsWithTypeAndName equal_columns(tuple_size - 1); ColumnsWithTypeAndName tmp_columns(2); diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index e03b27b3c39..5e1e7067e86 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -668,6 +668,9 @@ private: temporary_columns[0] = arguments[0]; size_t tuple_size = type1.getElements().size(); + if (tuple_size == 0) + return ColumnTuple::create(input_rows_count); + Columns tuple_columns(tuple_size); for (size_t i = 0; i < tuple_size; ++i) diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 6ed02212095..b66b9867e39 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -61,6 +61,7 @@ struct ReadSettings bool filesystem_cache_allow_background_download = true; bool filesystem_cache_allow_background_download_for_metadata_files_in_packed_storage = true; bool filesystem_cache_allow_background_download_during_fetch = true; + bool filesystem_cache_prefer_bigger_buffer_size = true; bool use_page_cache_for_disks_without_file_cache = false; bool read_from_page_cache_if_exists_otherwise_bypass_cache = false; @@ -68,7 +69,7 @@ struct ReadSettings std::shared_ptr page_cache; size_t filesystem_cache_max_download_size = (128UL * 1024 * 1024 * 1024); - bool skip_download_if_exceeds_query_cache = true; + bool filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit = true; size_t remote_read_min_bytes_for_seek = DBMS_DEFAULT_BUFFER_SIZE; diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 7c6a21941eb..aefe3ff338c 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -37,7 +37,7 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax) /// Case when bucket name represented in domain name of S3 URL. /// E.g. (https://bucket-name.s3.region.amazonaws.com/key) /// https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#virtual-hosted-style-access - static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss|eos)([.\-][a-z0-9\-.:]+))"); + static const RE2 virtual_hosted_style_pattern(R"((.+)\.(s3express[\-a-z0-9]+|s3|cos|obs|oss-data-acc|oss|eos)([.\-][a-z0-9\-.:]+))"); /// Case when AWS Private Link Interface is being used /// E.g. (bucket.vpce-07a1cd78f1bd55c5f-j3a3vg6w.s3.us-east-1.vpce.amazonaws.com/bucket-name/key) @@ -115,7 +115,15 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax) && re2::RE2::FullMatch(uri.getAuthority(), virtual_hosted_style_pattern, &bucket, &name, &endpoint_authority_from_uri)) { is_virtual_hosted_style = true; - endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri; + if (name == "oss-data-acc") + { + bucket = bucket.substr(0, bucket.find('.')); + endpoint = uri.getScheme() + "://" + uri.getHost().substr(bucket.length() + 1); + } + else + { + endpoint = uri.getScheme() + "://" + name + endpoint_authority_from_uri; + } validateBucket(bucket, uri); if (!uri.getPath().empty()) diff --git a/src/IO/tests/gtest_s3_uri.cpp b/src/IO/tests/gtest_s3_uri.cpp index 8696fab0616..6167313b634 100644 --- a/src/IO/tests/gtest_s3_uri.cpp +++ b/src/IO/tests/gtest_s3_uri.cpp @@ -212,6 +212,22 @@ TEST(S3UriTest, validPatterns) ASSERT_EQ("", uri.version_id); ASSERT_EQ(true, uri.is_virtual_hosted_style); } + { + S3::URI uri("https://bucket-test1.oss-cn-beijing-internal.aliyuncs.com/ab-test"); + ASSERT_EQ("https://oss-cn-beijing-internal.aliyuncs.com", uri.endpoint); + ASSERT_EQ("bucket-test1", uri.bucket); + ASSERT_EQ("ab-test", uri.key); + ASSERT_EQ("", uri.version_id); + ASSERT_EQ(true, uri.is_virtual_hosted_style); + } + { + S3::URI uri("https://bucket-test.cn-beijing-internal.oss-data-acc.aliyuncs.com/ab-test"); + ASSERT_EQ("https://cn-beijing-internal.oss-data-acc.aliyuncs.com", uri.endpoint); + ASSERT_EQ("bucket-test", uri.bucket); + ASSERT_EQ("ab-test", uri.key); + ASSERT_EQ("", uri.version_id); + ASSERT_EQ(true, uri.is_virtual_hosted_style); + } } TEST(S3UriTest, versionIdChecks) diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 7de3f7af78d..8887165a75d 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -53,16 +54,6 @@ namespace ErrorCodes namespace { - size_t roundDownToMultiple(size_t num, size_t multiple) - { - return (num / multiple) * multiple; - } - - size_t roundUpToMultiple(size_t num, size_t multiple) - { - return roundDownToMultiple(num + multiple - 1, multiple); - } - std::string getCommonUserID() { auto user_from_context = DB::Context::getGlobalContextInstance()->getFilesystemCacheUser(); @@ -96,6 +87,7 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s : max_file_segment_size(settings.max_file_segment_size) , bypass_cache_threshold(settings.enable_bypass_cache_with_threshold ? settings.bypass_cache_threshold : 0) , boundary_alignment(settings.boundary_alignment) + , background_download_max_file_segment_size(settings.background_download_max_file_segment_size) , load_metadata_threads(settings.load_metadata_threads) , load_metadata_asynchronously(settings.load_metadata_asynchronously) , write_cache_per_user_directory(settings.write_cache_per_user_id_directory) @@ -103,7 +95,10 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s , keep_current_elements_to_max_ratio(1 - settings.keep_free_space_elements_ratio) , keep_up_free_space_remove_batch(settings.keep_free_space_remove_batch) , log(getLogger("FileCache(" + cache_name + ")")) - , metadata(settings.base_path, settings.background_download_queue_size_limit, settings.background_download_threads, write_cache_per_user_directory) + , metadata(settings.base_path, + settings.background_download_queue_size_limit, + settings.background_download_threads, + write_cache_per_user_directory) { if (settings.cache_policy == "LRU") { @@ -601,8 +596,8 @@ FileCache::getOrSet( /// 2. max_file_segments_limit FileSegment::Range result_range = initial_range; - const auto aligned_offset = roundDownToMultiple(initial_range.left, boundary_alignment); - auto aligned_end_offset = std::min(roundUpToMultiple(initial_range.right + 1, boundary_alignment), file_size) - 1; + const auto aligned_offset = FileCacheUtils::roundDownToMultiple(initial_range.left, boundary_alignment); + auto aligned_end_offset = std::min(FileCacheUtils::roundUpToMultiple(initial_range.right + 1, boundary_alignment), file_size) - 1; chassert(aligned_offset <= initial_range.left); chassert(aligned_end_offset >= initial_range.right); @@ -1600,6 +1595,17 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings, } } + if (new_settings.background_download_max_file_segment_size != actual_settings.background_download_max_file_segment_size) + { + background_download_max_file_segment_size = new_settings.background_download_max_file_segment_size; + + LOG_INFO(log, "Changed background_download_max_file_segment_size from {} to {}", + actual_settings.background_download_max_file_segment_size, + new_settings.background_download_max_file_segment_size); + + actual_settings.background_download_max_file_segment_size = new_settings.background_download_max_file_segment_size; + } + if (new_settings.max_size != actual_settings.max_size || new_settings.max_elements != actual_settings.max_elements) { diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 810ed481300..bbe8502fec9 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -161,6 +161,10 @@ public: size_t getMaxFileSegmentSize() const { return max_file_segment_size; } + size_t getBackgroundDownloadMaxFileSegmentSize() const { return background_download_max_file_segment_size.load(); } + + size_t getBoundaryAlignment() const { return boundary_alignment; } + bool tryReserve( FileSegment & file_segment, size_t size, @@ -199,6 +203,7 @@ private: std::atomic max_file_segment_size; const size_t bypass_cache_threshold; const size_t boundary_alignment; + std::atomic background_download_max_file_segment_size; size_t load_metadata_threads; const bool load_metadata_asynchronously; std::atomic stop_loading_metadata = false; diff --git a/src/Interpreters/Cache/FileCacheSettings.cpp b/src/Interpreters/Cache/FileCacheSettings.cpp index e162d6b7551..8f0c5206211 100644 --- a/src/Interpreters/Cache/FileCacheSettings.cpp +++ b/src/Interpreters/Cache/FileCacheSettings.cpp @@ -62,6 +62,9 @@ void FileCacheSettings::loadImpl(FuncHas has, FuncGetUInt get_uint, FuncGetStrin if (has("background_download_queue_size_limit")) background_download_queue_size_limit = get_uint("background_download_queue_size_limit"); + if (has("background_download_max_file_segment_size")) + background_download_max_file_segment_size = get_uint("background_download_max_file_segment_size"); + if (has("load_metadata_threads")) load_metadata_threads = get_uint("load_metadata_threads"); diff --git a/src/Interpreters/Cache/FileCacheSettings.h b/src/Interpreters/Cache/FileCacheSettings.h index 72a2b6c3369..9cf72a2bdff 100644 --- a/src/Interpreters/Cache/FileCacheSettings.h +++ b/src/Interpreters/Cache/FileCacheSettings.h @@ -43,6 +43,8 @@ struct FileCacheSettings double keep_free_space_elements_ratio = FILECACHE_DEFAULT_FREE_SPACE_ELEMENTS_RATIO; size_t keep_free_space_remove_batch = FILECACHE_DEFAULT_FREE_SPACE_REMOVE_BATCH; + size_t background_download_max_file_segment_size = FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE_WITH_BACKGROUND_DOWLOAD; + void loadFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); void loadFromCollection(const NamedCollection & collection); diff --git a/src/Interpreters/Cache/FileCacheUtils.h b/src/Interpreters/Cache/FileCacheUtils.h new file mode 100644 index 00000000000..b35ce867a79 --- /dev/null +++ b/src/Interpreters/Cache/FileCacheUtils.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace FileCacheUtils +{ + +static size_t roundDownToMultiple(size_t num, size_t multiple) +{ + return (num / multiple) * multiple; +} + +static size_t roundUpToMultiple(size_t num, size_t multiple) +{ + return roundDownToMultiple(num + multiple - 1, multiple); +} + +} diff --git a/src/Interpreters/Cache/FileCache_fwd.h b/src/Interpreters/Cache/FileCache_fwd.h index da75f30f0e8..3d461abd065 100644 --- a/src/Interpreters/Cache/FileCache_fwd.h +++ b/src/Interpreters/Cache/FileCache_fwd.h @@ -6,6 +6,7 @@ namespace DB static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi +static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE_WITH_BACKGROUND_DOWLOAD = 4 * 1024 * 1024; /// 4Mi static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 5; static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_QUEUE_SIZE_LIMIT = 5000; static constexpr int FILECACHE_DEFAULT_LOAD_METADATA_THREADS = 16; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 541f0f5607a..74148aa5461 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -360,11 +361,14 @@ void FileSegment::write(char * from, size_t size, size_t offset_in_file) "Expected DOWNLOADING state, got {}", stateToString(download_state)); const size_t first_non_downloaded_offset = getCurrentWriteOffset(); + if (offset_in_file != first_non_downloaded_offset) + { throw Exception( ErrorCodes::LOGICAL_ERROR, "Attempt to write {} bytes to offset: {}, but current write offset is {}", size, offset_in_file, first_non_downloaded_offset); + } const size_t current_downloaded_size = getDownloadedSize(); chassert(reserved_size >= current_downloaded_size); @@ -375,8 +379,19 @@ void FileSegment::write(char * from, size_t size, size_t offset_in_file) ErrorCodes::LOGICAL_ERROR, "Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size); - if (!is_unbound && current_downloaded_size == range().size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); + if (!is_unbound) + { + if (current_downloaded_size == range().size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); + + if (current_downloaded_size + size > range().size()) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot download beyond file segment boundaries: {}. Write offset: {}, size: {}, downloaded size: {}", + range().size(), first_non_downloaded_offset, size, current_downloaded_size); + } + } if (!cache_writer && current_downloaded_size > 0) throw Exception( @@ -629,6 +644,36 @@ void FileSegment::completePartAndResetDownloader() LOG_TEST(log, "Complete batch. ({})", getInfoForLogUnlocked(lk)); } +size_t FileSegment::getSizeForBackgroundDownload() const +{ + auto lk = lock(); + return getSizeForBackgroundDownloadUnlocked(lk); +} + +size_t FileSegment::getSizeForBackgroundDownloadUnlocked(const FileSegmentGuard::Lock &) const +{ + if (!background_download_enabled + || !downloaded_size + || !remote_file_reader) + { + return 0; + } + + chassert(downloaded_size <= range().size()); + + const size_t background_download_max_file_segment_size = cache->getBackgroundDownloadMaxFileSegmentSize(); + size_t desired_size; + if (downloaded_size >= background_download_max_file_segment_size) + desired_size = FileCacheUtils::roundUpToMultiple(downloaded_size, cache->getBoundaryAlignment()); + else + desired_size = FileCacheUtils::roundUpToMultiple(background_download_max_file_segment_size, cache->getBoundaryAlignment()); + + desired_size = std::min(desired_size, range().size()); + chassert(desired_size >= downloaded_size); + + return desired_size - downloaded_size; +} + void FileSegment::complete(bool allow_background_download) { ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentCompleteMicroseconds); @@ -708,7 +753,8 @@ void FileSegment::complete(bool allow_background_download) if (is_last_holder) { bool added_to_download_queue = false; - if (allow_background_download && background_download_enabled && remote_file_reader) + size_t background_download_size = allow_background_download ? getSizeForBackgroundDownloadUnlocked(segment_lock) : 0; + if (background_download_size) { ProfileEvents::increment(ProfileEvents::FilesystemCacheBackgroundDownloadQueuePush); added_to_download_queue = locked_key->addToDownloadQueue(offset(), segment_lock); /// Finish download in background. @@ -862,7 +908,12 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) chassert(downloaded_size == reserved_size); chassert(downloaded_size == range().size()); chassert(downloaded_size > 0); - chassert(fs::file_size(getPath()) > 0); + + auto file_size = fs::file_size(getPath()); + UNUSED(file_size); + + chassert(file_size == range().size()); + chassert(downloaded_size == range().size()); chassert(queue_iterator || on_delayed_removal); check_iterator(queue_iterator); @@ -884,7 +935,13 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) chassert(reserved_size >= downloaded_size); chassert(downloaded_size > 0); - chassert(fs::file_size(getPath()) > 0); + + auto file_size = fs::file_size(getPath()); + UNUSED(file_size); + + chassert(file_size > 0); + chassert(file_size <= range().size()); + chassert(downloaded_size <= range().size()); chassert(queue_iterator); check_iterator(queue_iterator); diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 21d5f9dab5f..a6bfb203cec 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -185,6 +185,8 @@ public: bool assertCorrectness() const; + size_t getSizeForBackgroundDownload() const; + /** * ========== Methods that must do cv.notify() ================== */ @@ -230,6 +232,7 @@ private: String getDownloaderUnlocked(const FileSegmentGuard::Lock &) const; bool isDownloaderUnlocked(const FileSegmentGuard::Lock & segment_lock) const; void resetDownloaderUnlocked(const FileSegmentGuard::Lock &); + size_t getSizeForBackgroundDownloadUnlocked(const FileSegmentGuard::Lock &) const; void setDownloadState(State state, const FileSegmentGuard::Lock &); void resetDownloadingStateUnlocked(const FileSegmentGuard::Lock &); diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index 231545212cd..4d3033191dc 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -676,13 +676,17 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optionalinternalBuffer().empty()) { if (!memory) - memory.emplace(DBMS_DEFAULT_BUFFER_SIZE); + memory.emplace(std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), size_to_download)); reader->set(memory->data(), memory->size()); } @@ -701,9 +705,13 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optional(reader->getPosition())) reader->seek(offset, SEEK_SET); - while (!reader->eof()) + while (size_to_download && !reader->eof()) { - auto size = reader->available(); + const auto available = reader->available(); + chassert(available); + + const auto size = std::min(available, size_to_download); + size_to_download -= size; std::string failure_reason; if (!file_segment.reserve(size, reserve_space_lock_wait_timeout_milliseconds, failure_reason)) @@ -713,7 +721,7 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optional LockedKey::sync() actual_size, expected_size, file_segment->getInfoForLog()); broken.push_back(FileSegment::getInfo(file_segment)); - it = removeFileSegment(file_segment->offset(), file_segment->lock(), /* can_be_broken */false); + it = removeFileSegment(file_segment->offset(), file_segment->lock(), /* can_be_broken */true); } return broken; } diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index 0e85ead3265..24683b2de71 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -210,6 +210,7 @@ public: bool setBackgroundDownloadThreads(size_t threads_num); size_t getBackgroundDownloadThreads() const { return download_threads.size(); } + bool setBackgroundDownloadQueueSizeLimit(size_t size); bool isBackgroundDownloadEnabled(); diff --git a/src/Interpreters/Cache/QueryLimit.cpp b/src/Interpreters/Cache/QueryLimit.cpp index b18d23a5b7f..a7c964022a5 100644 --- a/src/Interpreters/Cache/QueryLimit.cpp +++ b/src/Interpreters/Cache/QueryLimit.cpp @@ -53,7 +53,7 @@ FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::getOrSetQueryContext( { it->second = std::make_shared( settings.filesystem_cache_max_download_size, - !settings.skip_download_if_exceeds_query_cache); + !settings.filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit); } return it->second; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c1fa2c8549a..d2aad0a52d8 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -196,6 +196,7 @@ namespace Setting extern const SettingsUInt64 filesystem_cache_segments_batch_size; extern const SettingsBool filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage; extern const SettingsBool filesystem_cache_enable_background_download_during_fetch; + extern const SettingsBool filesystem_cache_prefer_bigger_buffer_size; extern const SettingsBool http_make_head_request; extern const SettingsUInt64 http_max_fields; extern const SettingsUInt64 http_max_field_name_size; @@ -236,7 +237,7 @@ namespace Setting extern const SettingsUInt64 remote_fs_read_backoff_max_tries; extern const SettingsUInt64 remote_read_min_bytes_for_seek; extern const SettingsBool throw_on_error_from_cache_on_write_operations; - extern const SettingsBool skip_download_if_exceeds_query_cache; + extern const SettingsBool filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit; extern const SettingsBool s3_allow_parallel_part_upload; extern const SettingsBool use_page_cache_for_disks_without_file_cache; extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; @@ -5751,9 +5752,10 @@ ReadSettings Context::getReadSettings() const res.filesystem_cache_allow_background_download_for_metadata_files_in_packed_storage = settings_ref[Setting::filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage]; res.filesystem_cache_allow_background_download_during_fetch = settings_ref[Setting::filesystem_cache_enable_background_download_during_fetch]; + res.filesystem_cache_prefer_bigger_buffer_size = settings_ref[Setting::filesystem_cache_prefer_bigger_buffer_size]; res.filesystem_cache_max_download_size = settings_ref[Setting::filesystem_cache_max_download_size]; - res.skip_download_if_exceeds_query_cache = settings_ref[Setting::skip_download_if_exceeds_query_cache]; + res.filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit = settings_ref[Setting::filesystem_cache_skip_download_if_exceeds_per_query_cache_write_limit]; res.page_cache = getPageCache(); res.use_page_cache_for_disks_without_file_cache = settings_ref[Setting::use_page_cache_for_disks_without_file_cache]; diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index d4da038c089..afab06a13ba 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -19,9 +21,8 @@ #include #include #include -#include #include -#include +#include namespace ProfileEvents { @@ -67,6 +68,18 @@ bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr return false; } + if (auto * tables = node->as()) + { + /// Contrary to what's said in the code block above, ARRAY JOIN needs to resolve the subquery if possible + /// and assign an alias for 02367_optimize_trivial_count_with_array_join to pass. Otherwise it will fail in + /// ArrayJoinedColumnsVisitor (`No alias for non-trivial value in ARRAY JOIN: _a`) + /// This looks 100% as a incomplete code working on top of a bug, but this code has already been made obsolete + /// by the new analyzer, so it's an inconvenience we can live with until we deprecate it. + if (child == tables->array_join) + return true; + return false; + } + return true; } @@ -246,6 +259,8 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr if (tmp_block.rows() != 0) throw Exception(ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY, "Scalar subquery returned more than one row"); + + logProcessorProfile(data.getContext(), io.pipeline.getProcessors()); } block = materializeBlock(block); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f6586f8bfc2..b027f6aad61 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -98,6 +98,9 @@ namespace CurrentMetrics { extern const Metric AttachedTable; + extern const Metric AttachedReplicatedTable; + extern const Metric AttachedDictionary; + extern const Metric AttachedView; } namespace DB @@ -145,7 +148,10 @@ namespace ServerSetting { extern const ServerSettingsBool ignore_empty_sql_security_in_create_view_query; extern const ServerSettingsUInt64 max_database_num_to_throw; + extern const ServerSettingsUInt64 max_dictionary_num_to_throw; extern const ServerSettingsUInt64 max_table_num_to_throw; + extern const ServerSettingsUInt64 max_replicated_table_num_to_throw; + extern const ServerSettingsUInt64 max_view_num_to_throw; } namespace ErrorCodes @@ -1912,16 +1918,8 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, } } - UInt64 table_num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_table_num_to_throw]; - if (table_num_limit > 0 && !internal) - { - UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable); - if (table_count >= table_num_limit) - throw Exception(ErrorCodes::TOO_MANY_TABLES, - "Too many tables. " - "The limit (server configuration parameter `max_table_num_to_throw`) is set to {}, the current number of tables is {}", - table_num_limit, table_count); - } + if (!internal) + throwIfTooManyEntities(create, res); database->createTable(getContext(), create.getTable(), res, query_ptr); @@ -1948,6 +1946,30 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, } +void InterpreterCreateQuery::throwIfTooManyEntities(ASTCreateQuery & create, StoragePtr storage) const +{ + auto check_and_throw = [&](auto setting, CurrentMetrics::Metric metric, String setting_name, String entity_name) + { + UInt64 num_limit = getContext()->getGlobalContext()->getServerSettings()[setting]; + UInt64 attached_count = CurrentMetrics::get(metric); + if (num_limit > 0 && attached_count >= num_limit) + throw Exception(ErrorCodes::TOO_MANY_TABLES, + "Too many {}. " + "The limit (server configuration parameter `{}`) is set to {}, the current number is {}", + entity_name, setting_name, num_limit, attached_count); + }; + + if (auto * replicated_storage = typeid_cast(storage.get())) + check_and_throw(ServerSetting::max_replicated_table_num_to_throw, CurrentMetrics::AttachedReplicatedTable, "max_replicated_table_num_to_throw", "replicated tables"); + else if (create.is_dictionary) + check_and_throw(ServerSetting::max_dictionary_num_to_throw, CurrentMetrics::AttachedDictionary, "max_dictionary_num_to_throw", "dictionaries"); + else if (create.isView()) + check_and_throw(ServerSetting::max_view_num_to_throw, CurrentMetrics::AttachedView, "max_view_num_to_throw", "views"); + else + check_and_throw(ServerSetting::max_table_num_to_throw, CurrentMetrics::AttachedTable, "max_table_num_to_throw", "tables"); +} + + BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties, LoadingStrictnessLevel mode) { diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index cb7af25383e..24cf308951c 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -122,6 +122,8 @@ private: BlockIO executeQueryOnCluster(ASTCreateQuery & create); + void throwIfTooManyEntities(ASTCreateQuery & create, StoragePtr storage) const; + ASTPtr query_ptr; /// Skip safety threshold when loading tables. diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 0f25d5ac21c..a35353a6b2a 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -53,6 +53,7 @@ namespace Setting extern const SettingsBool allow_nondeterministic_mutations; extern const SettingsUInt64 max_block_size; extern const SettingsBool use_concurrency_control; + extern const SettingsBool validate_mutation_query; } namespace MergeTreeSetting @@ -1386,6 +1387,18 @@ void MutationsInterpreter::validate() } } + // Make sure the mutation query is valid + if (context->getSettingsRef()[Setting::validate_mutation_query]) + { + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) + prepareQueryAffectedQueryTree(commands, source.getStorage(), context); + else + { + ASTPtr select_query = prepareQueryAffectedAST(commands, source.getStorage(), context); + InterpreterSelectQuery(select_query, context, source.getStorage(), metadata_snapshot); + } + } + QueryPlan plan; initQueryPlan(stages.front(), plan); diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 538108165fb..c69e2f84d42 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -1,21 +1,22 @@ #include #include -#include -#include -#include -#include -#include +#include #include -#include +#include +#include +#include +#include +#include #include #include +#include #include +#include #include #include -#include -#include #include #include +#include namespace DB { @@ -239,6 +240,8 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) if (!set_and_key->set->isCreated()) return nullptr; + logProcessorProfile(context, pipeline.getProcessors()); + return set_and_key->set; } diff --git a/src/Interpreters/ProcessorsProfileLog.cpp b/src/Interpreters/ProcessorsProfileLog.cpp index 8a646b5d0e7..d7811e5e9e2 100644 --- a/src/Interpreters/ProcessorsProfileLog.cpp +++ b/src/Interpreters/ProcessorsProfileLog.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -8,16 +9,19 @@ #include #include #include +#include #include #include #include -#include - -#include namespace DB { +namespace Setting +{ +extern const SettingsBool log_processors_profiles; +} + ColumnsDescription ProcessorProfileLogElement::getColumnsDescription() { return ColumnsDescription @@ -81,5 +85,57 @@ void ProcessorProfileLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(output_bytes); } +void logProcessorProfile(ContextPtr context, const Processors & processors) +{ + const Settings & settings = context->getSettingsRef(); + if (settings[Setting::log_processors_profiles]) + { + if (auto processors_profile_log = context->getProcessorsProfileLog()) + { + ProcessorProfileLogElement processor_elem; + const auto time_now = std::chrono::system_clock::now(); + processor_elem.event_time = timeInSeconds(time_now); + processor_elem.event_time_microseconds = timeInMicroseconds(time_now); + processor_elem.initial_query_id = context->getInitialQueryId(); + processor_elem.query_id = context->getCurrentQueryId(); + + auto get_proc_id = [](const IProcessor & proc) -> UInt64 { return reinterpret_cast(&proc); }; + + for (const auto & processor : processors) + { + std::vector parents; + for (const auto & port : processor->getOutputs()) + { + if (!port.isConnected()) + continue; + const IProcessor & next = port.getInputPort().getProcessor(); + parents.push_back(get_proc_id(next)); + } + + processor_elem.id = get_proc_id(*processor); + processor_elem.parent_ids = std::move(parents); + + processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); + processor_elem.plan_step_name = processor->getPlanStepName(); + processor_elem.plan_step_description = processor->getPlanStepDescription(); + processor_elem.plan_group = processor->getQueryPlanStepGroup(); + + processor_elem.processor_name = processor->getName(); + + processor_elem.elapsed_us = static_cast(processor->getElapsedNs() / 1000U); + processor_elem.input_wait_elapsed_us = static_cast(processor->getInputWaitElapsedNs() / 1000U); + processor_elem.output_wait_elapsed_us = static_cast(processor->getOutputWaitElapsedNs() / 1000U); + + auto stats = processor->getProcessorDataStats(); + processor_elem.input_rows = stats.input_rows; + processor_elem.input_bytes = stats.input_bytes; + processor_elem.output_rows = stats.output_rows; + processor_elem.output_bytes = stats.output_bytes; + + processors_profile_log->add(processor_elem); + } + } + } +} } diff --git a/src/Interpreters/ProcessorsProfileLog.h b/src/Interpreters/ProcessorsProfileLog.h index fbf52f45f56..9cc2ab6c7f0 100644 --- a/src/Interpreters/ProcessorsProfileLog.h +++ b/src/Interpreters/ProcessorsProfileLog.h @@ -50,4 +50,5 @@ public: using SystemLog::SystemLog; }; +void logProcessorProfile(ContextPtr context, const Processors & processors); } diff --git a/src/Interpreters/QueryMetricLog.cpp b/src/Interpreters/QueryMetricLog.cpp index 5ab3fe590e0..52d773b7d1b 100644 --- a/src/Interpreters/QueryMetricLog.cpp +++ b/src/Interpreters/QueryMetricLog.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include -#include namespace DB @@ -24,6 +24,15 @@ namespace DB static auto logger = getLogger("QueryMetricLog"); +String timePointToString(QueryMetricLog::TimePoint time) +{ + /// fmtlib supports subsecond formatting in 10.0.0. We're in 9.1.0, so we need to add the milliseconds ourselves. + auto seconds = std::chrono::time_point_cast(time); + auto microseconds = std::chrono::duration_cast(time - seconds).count(); + + return fmt::format("{:%Y.%m.%d %H:%M:%S}.{:06}", seconds, microseconds); +} + ColumnsDescription QueryMetricLogElement::getColumnsDescription() { ColumnsDescription result; @@ -87,36 +96,73 @@ void QueryMetricLog::shutdown() Base::shutdown(); } -void QueryMetricLog::startQuery(const String & query_id, TimePoint start_time, UInt64 interval_milliseconds) +void QueryMetricLog::collectMetric(const ProcessList & process_list, String query_id) { - QueryMetricLogStatus status; - status.interval_milliseconds = interval_milliseconds; - status.next_collect_time = start_time + std::chrono::milliseconds(interval_milliseconds); + auto current_time = std::chrono::system_clock::now(); + const auto query_info = process_list.getQueryInfo(query_id, false, true, false); + if (!query_info) + { + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Query {} is not running anymore, so we couldn't get its QueryStatusInfo", query_id); + return; + } + + LockGuard global_lock(queries_mutex); + auto it = queries.find(query_id); + + /// The query might have finished while the scheduled task is running. + if (it == queries.end()) + { + global_lock.unlock(); + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Query {} not found in the list. Finished while this collecting task was running", query_id); + return; + } + + auto & query_status = it->second; + if (!query_status.mutex) + { + global_lock.unlock(); + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Query {} finished while this collecting task was running", query_id); + return; + } + + LockGuard query_lock(query_status.getMutex()); + global_lock.unlock(); + + auto elem = query_status.createLogMetricElement(query_id, *query_info, current_time); + if (elem) + add(std::move(elem.value())); +} + +/// We use TSA_NO_THREAD_SAFETY_ANALYSIS to prevent TSA complaining that we're modifying the query_status fields +/// without locking the mutex. Since we're building it from scratch, there's no harm in not holding it. +/// If we locked it to make TSA happy, TSAN build would falsely complain about +/// lock-order-inversion (potential deadlock) +/// which is not a real issue since QueryMetricLogStatus's mutex cannot be locked by anything else +/// until we add it to the queries map. +void QueryMetricLog::startQuery(const String & query_id, TimePoint start_time, UInt64 interval_milliseconds) TSA_NO_THREAD_SAFETY_ANALYSIS +{ + QueryMetricLogStatus query_status; + QueryMetricLogStatusInfo & info = query_status.info; + info.interval_milliseconds = interval_milliseconds; + info.next_collect_time = start_time; auto context = getContext(); const auto & process_list = context->getProcessList(); - status.task = context->getSchedulePool().createTask("QueryMetricLog", [this, &process_list, query_id] { - auto current_time = std::chrono::system_clock::now(); - const auto query_info = process_list.getQueryInfo(query_id, false, true, false); - if (!query_info) - { - LOG_TRACE(logger, "Query {} is not running anymore, so we couldn't get its QueryStatusInfo", query_id); - return; - } - - auto elem = createLogMetricElement(query_id, *query_info, current_time); - if (elem) - add(std::move(elem.value())); + info.task = context->getSchedulePool().createTask("QueryMetricLog", [this, &process_list, query_id] { + collectMetric(process_list, query_id); }); - std::lock_guard lock(queries_mutex); - status.task->scheduleAfter(interval_milliseconds); - queries.emplace(query_id, std::move(status)); + LockGuard global_lock(queries_mutex); + query_status.scheduleNext(query_id); + queries.emplace(query_id, std::move(query_status)); } void QueryMetricLog::finishQuery(const String & query_id, TimePoint finish_time, QueryStatusInfoPtr query_info) { - std::unique_lock lock(queries_mutex); + LockGuard global_lock(queries_mutex); auto it = queries.find(query_id); /// finishQuery may be called from logExceptionBeforeStart when the query has not even started @@ -124,9 +170,19 @@ void QueryMetricLog::finishQuery(const String & query_id, TimePoint finish_time, if (it == queries.end()) return; + auto & query_status = it->second; + decltype(query_status.mutex) query_mutex; + LockGuard query_lock(query_status.getMutex()); + + /// Move the query mutex here so that we hold it until the end, after removing the query from queries. + query_mutex = std::move(query_status.mutex); + query_status.mutex = {}; + + global_lock.unlock(); + if (query_info) { - auto elem = createLogMetricElement(query_id, *query_info, finish_time, false); + auto elem = query_status.createLogMetricElement(query_id, *query_info, finish_time, false); if (elem) add(std::move(elem.value())); } @@ -139,51 +195,58 @@ void QueryMetricLog::finishQuery(const String & query_id, TimePoint finish_time, /// that order. { /// Take ownership of the task so that we can destroy it in this scope after unlocking `queries_mutex`. - auto task = std::move(it->second.task); + auto task = std::move(query_status.info.task); /// Build an empty task for the old task to make sure it does not lock any mutex on its destruction. - it->second.task = {}; + query_status.info.task = {}; + query_lock.unlock(); + global_lock.lock(); queries.erase(query_id); /// Ensure `queries_mutex` is unlocked before calling task's destructor at the end of this /// scope which will lock `exec_mutex`. - lock.unlock(); + global_lock.unlock(); } } -std::optional QueryMetricLog::createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next) +void QueryMetricLogStatus::scheduleNext(String query_id) { - /// fmtlib supports subsecond formatting in 10.0.0. We're in 9.1.0, so we need to add the milliseconds ourselves. - auto seconds = std::chrono::time_point_cast(query_info_time); - auto microseconds = std::chrono::duration_cast(query_info_time - seconds).count(); - LOG_DEBUG(logger, "Collecting query_metric_log for query {} with QueryStatusInfo from {:%Y.%m.%d %H:%M:%S}.{:06}. Schedule next: {}", query_id, seconds, microseconds, schedule_next); - - std::unique_lock lock(queries_mutex); - auto query_status_it = queries.find(query_id); - - /// The query might have finished while the scheduled task is running. - if (query_status_it == queries.end()) + info.next_collect_time += std::chrono::milliseconds(info.interval_milliseconds); + const auto now = std::chrono::system_clock::now(); + if (info.next_collect_time > now) { - lock.unlock(); - LOG_TRACE(logger, "Query {} finished already while this collecting task was running", query_id); - return {}; + const auto wait_time = std::chrono::duration_cast(info.next_collect_time - now).count(); + info.task->scheduleAfter(wait_time); } - - auto & query_status = query_status_it->second; - if (query_info_time <= query_status.last_collect_time) + else { - lock.unlock(); + LOG_TRACE(logger, "The next collecting task for query {} should have already run at {}. Scheduling it right now", + query_id, timePointToString(info.next_collect_time)); + info.task->schedule(); + } +} + +std::optional QueryMetricLogStatus::createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next) +{ + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_TRACE(logger, "Collecting query_metric_log for query {} and interval {} ms with QueryStatusInfo from {}. Next collection time: {}", + query_id, info.interval_milliseconds, timePointToString(query_info_time), + schedule_next ? timePointToString(info.next_collect_time + std::chrono::milliseconds(info.interval_milliseconds)) : "finished"); + + if (query_info_time <= info.last_collect_time) + { + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI LOG_TRACE(logger, "Query {} has a more recent metrics collected. Skipping this one", query_id); return {}; } - query_status.last_collect_time = query_info_time; + info.last_collect_time = query_info_time; QueryMetricLogElement elem; elem.event_time = timeInSeconds(query_info_time); elem.event_time_microseconds = timeInMicroseconds(query_info_time); - elem.query_id = query_status_it->first; + elem.query_id = query_id; elem.memory_usage = query_info.memory_usage > 0 ? query_info.memory_usage : 0; elem.peak_memory_usage = query_info.peak_memory_usage > 0 ? query_info.peak_memory_usage : 0; @@ -192,7 +255,7 @@ std::optional QueryMetricLog::createLogMetricElement(cons for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i) { const auto & new_value = (*(query_info.profile_counters))[i]; - auto & old_value = query_status.last_profile_events[i]; + auto & old_value = info.last_profile_events[i]; /// Profile event counters are supposed to be monotonic. However, at least the `NetworkReceiveBytes` can be inaccurate. /// So, since in the future the counter should always have a bigger value than in the past, we skip this event. @@ -208,16 +271,13 @@ std::optional QueryMetricLog::createLogMetricElement(cons } else { - LOG_TRACE(logger, "Query {} has no profile counters", query_id); + /// TODO: remove trace before 24.11 release after checking everything is fine on the CI + LOG_DEBUG(logger, "Query {} has no profile counters", query_id); elem.profile_events = std::vector(ProfileEvents::end()); } if (schedule_next) - { - query_status.next_collect_time += std::chrono::milliseconds(query_status.interval_milliseconds); - const auto wait_time = std::chrono::duration_cast(query_status.next_collect_time - std::chrono::system_clock::now()).count(); - query_status.task->scheduleAfter(wait_time); - } + scheduleNext(query_id); return elem; } diff --git a/src/Interpreters/QueryMetricLog.h b/src/Interpreters/QueryMetricLog.h index 802cee7bf26..9371dfbb6b5 100644 --- a/src/Interpreters/QueryMetricLog.h +++ b/src/Interpreters/QueryMetricLog.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -11,11 +12,17 @@ #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +}; + /** QueryMetricLogElement is a log of query metric values measured at regular time interval. */ @@ -34,7 +41,7 @@ struct QueryMetricLogElement void appendToBlock(MutableColumns & columns) const; }; -struct QueryMetricLogStatus +struct QueryMetricLogStatusInfo { UInt64 interval_milliseconds; std::chrono::system_clock::time_point last_collect_time; @@ -43,24 +50,47 @@ struct QueryMetricLogStatus BackgroundSchedulePool::TaskHolder task; }; +struct QueryMetricLogStatus +{ + using TimePoint = std::chrono::system_clock::time_point; + using Mutex = std::mutex; + + QueryMetricLogStatusInfo info TSA_GUARDED_BY(getMutex()); + + /// We need to be able to move it for the hash map, so we need to add an indirection here. + std::unique_ptr mutex = std::make_unique(); + + /// Return a reference to the mutex, used for Thread Sanitizer annotations. + Mutex & getMutex() const TSA_RETURN_CAPABILITY(mutex) + { + if (!mutex) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mutex cannot be NULL"); + return *mutex; + } + + void scheduleNext(String query_id) TSA_REQUIRES(getMutex()); + std::optional createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next = true) TSA_REQUIRES(getMutex()); +}; + class QueryMetricLog : public SystemLog { using SystemLog::SystemLog; - using TimePoint = std::chrono::system_clock::time_point; using Base = SystemLog; public: + using TimePoint = std::chrono::system_clock::time_point; + void shutdown() final; - // Both startQuery and finishQuery are called from the thread that executes the query + /// Both startQuery and finishQuery are called from the thread that executes the query. void startQuery(const String & query_id, TimePoint start_time, UInt64 interval_milliseconds); void finishQuery(const String & query_id, TimePoint finish_time, QueryStatusInfoPtr query_info = nullptr); private: - std::optional createLogMetricElement(const String & query_id, const QueryStatusInfo & query_info, TimePoint query_info_time, bool schedule_next = true); + void collectMetric(const ProcessList & process_list, String query_id); - std::recursive_mutex queries_mutex; - std::unordered_map queries; + std::mutex queries_mutex; + std::unordered_map queries TSA_GUARDED_BY(queries_mutex); }; } diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index a8639906aad..bba30fb5194 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -161,7 +161,13 @@ void QueryNormalizer::visit(ASTTablesInSelectQueryElement & node, const ASTPtr & { auto & join = node.table_join->as(); if (join.on_expression) + { + ASTPtr original_on_expression = join.on_expression; visit(join.on_expression, data); + if (join.on_expression != original_on_expression) + join.children = { join.on_expression }; + } + } } diff --git a/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp b/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp index 22ce91d8c67..2890357494d 100644 --- a/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp +++ b/src/Interpreters/RewriteArrayExistsFunctionVisitor.cpp @@ -6,6 +6,12 @@ namespace DB { + +namespace ErrorCode +{ +extern const int LOGICAL_ERROR; +} + void RewriteArrayExistsFunctionMatcher::visit(ASTPtr & ast, Data & data) { if (auto * func = ast->as()) @@ -20,21 +26,21 @@ void RewriteArrayExistsFunctionMatcher::visit(ASTPtr & ast, Data & data) if (join->using_expression_list) { auto * it = std::find(join->children.begin(), join->children.end(), join->using_expression_list); + if (it == join->children.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not find join->using_expression_list in '{}'", join->formatForLogging()); visit(join->using_expression_list, data); - - if (it && *it != join->using_expression_list) - *it = join->using_expression_list; + *it = join->using_expression_list; } if (join->on_expression) { auto * it = std::find(join->children.begin(), join->children.end(), join->on_expression); + if (it == join->children.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not find join->on_expression in '{}'", join->formatForLogging()); visit(join->on_expression, data); - - if (it && *it != join->on_expression) - *it = join->on_expression; + *it = join->on_expression; } } } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 9250c069283..0bc1d4956a1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -117,7 +117,6 @@ namespace Setting extern const SettingsOverflowMode join_overflow_mode; extern const SettingsString log_comment; extern const SettingsBool log_formatted_queries; - extern const SettingsBool log_processors_profiles; extern const SettingsBool log_profile_events; extern const SettingsUInt64 log_queries_cut_to_length; extern const SettingsBool log_queries; @@ -506,6 +505,7 @@ void logQueryFinish( auto time_now = std::chrono::system_clock::now(); QueryStatusInfo info = process_list_elem->getInfo(true, settings[Setting::log_profile_events]); + logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, std::make_shared(info)); elem.type = QueryLogElementType::QUERY_FINISH; addStatusInfoToQueryLogElement(elem, info, query_ast, context); @@ -551,53 +551,8 @@ void logQueryFinish( if (auto query_log = context->getQueryLog()) query_log->add(elem); } - if (settings[Setting::log_processors_profiles]) - { - if (auto processors_profile_log = context->getProcessorsProfileLog()) - { - ProcessorProfileLogElement processor_elem; - processor_elem.event_time = elem.event_time; - processor_elem.event_time_microseconds = elem.event_time_microseconds; - processor_elem.initial_query_id = elem.client_info.initial_query_id; - processor_elem.query_id = elem.client_info.current_query_id; - auto get_proc_id = [](const IProcessor & proc) -> UInt64 { return reinterpret_cast(&proc); }; - - for (const auto & processor : query_pipeline.getProcessors()) - { - std::vector parents; - for (const auto & port : processor->getOutputs()) - { - if (!port.isConnected()) - continue; - const IProcessor & next = port.getInputPort().getProcessor(); - parents.push_back(get_proc_id(next)); - } - - processor_elem.id = get_proc_id(*processor); - processor_elem.parent_ids = std::move(parents); - - processor_elem.plan_step = reinterpret_cast(processor->getQueryPlanStep()); - processor_elem.plan_step_name = processor->getPlanStepName(); - processor_elem.plan_step_description = processor->getPlanStepDescription(); - processor_elem.plan_group = processor->getQueryPlanStepGroup(); - - processor_elem.processor_name = processor->getName(); - - processor_elem.elapsed_us = static_cast(processor->getElapsedNs() / 1000U); - processor_elem.input_wait_elapsed_us = static_cast(processor->getInputWaitElapsedNs() / 1000U); - processor_elem.output_wait_elapsed_us = static_cast(processor->getOutputWaitElapsedNs() / 1000U); - - auto stats = processor->getProcessorDataStats(); - processor_elem.input_rows = stats.input_rows; - processor_elem.input_bytes = stats.input_bytes; - processor_elem.output_rows = stats.output_rows; - processor_elem.output_bytes = stats.output_bytes; - - processors_profile_log->add(processor_elem); - } - } - } + logProcessorProfile(context, query_pipeline.getProcessors()); logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, std::make_shared(info)); } @@ -669,6 +624,7 @@ void logQueryException( { elem.query_duration_ms = start_watch.elapsedMilliseconds(); } + logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, info); elem.query_cache_usage = QueryCache::Usage::None; @@ -698,8 +654,6 @@ void logQueryException( query_span->addAttribute("clickhouse.exception_code", elem.exception_code); query_span->finish(); } - - logQueryMetricLogFinish(context, internal, elem.client_info.current_query_id, time_now, info); } void logExceptionBeforeStart( @@ -753,6 +707,8 @@ void logExceptionBeforeStart( elem.client_info = context->getClientInfo(); + logQueryMetricLogFinish(context, false, elem.client_info.current_query_id, std::chrono::system_clock::now(), nullptr); + elem.log_comment = settings[Setting::log_comment]; if (elem.log_comment.size() > settings[Setting::max_query_size]) elem.log_comment.resize(settings[Setting::max_query_size]); @@ -797,8 +753,6 @@ void logExceptionBeforeStart( ProfileEvents::increment(ProfileEvents::FailedInsertQuery); } } - - logQueryMetricLogFinish(context, false, elem.client_info.current_query_id, std::chrono::system_clock::now(), nullptr); } void validateAnalyzerSettings(ASTPtr ast, bool context_value) diff --git a/src/Parsers/ParserPartition.cpp b/src/Parsers/ParserPartition.cpp index ab97b3d0e3b..55b0d5a3567 100644 --- a/src/Parsers/ParserPartition.cpp +++ b/src/Parsers/ParserPartition.cpp @@ -5,10 +5,10 @@ #include #include #include -#include #include #include + namespace DB { @@ -61,7 +61,7 @@ bool ParserPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) else fields_count = 0; } - else if (const auto* literal_ast = value->as(); literal_ast) + else if (const auto * literal_ast = value->as(); literal_ast) { if (literal_ast->value.getType() == Field::Types::Tuple) { diff --git a/src/Parsers/StringRange.h b/src/Parsers/StringRange.h deleted file mode 100644 index 3c5959fd0a3..00000000000 --- a/src/Parsers/StringRange.h +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - - -namespace DB -{ - -struct StringRange -{ - const char * first = nullptr; - const char * second = nullptr; - - StringRange() = default; - StringRange(const char * begin, const char * end) : first(begin), second(end) {} - explicit StringRange(TokenIterator token) : first(token->begin), second(token->end) {} - - StringRange(TokenIterator token_begin, TokenIterator token_end) - { - /// Empty range. - if (token_begin == token_end) - { - first = token_begin->begin; - second = token_begin->begin; - return; - } - - TokenIterator token_last = token_end; - --token_last; - - first = token_begin->begin; - second = token_last->end; - } -}; - -using StringPtr = std::shared_ptr; - - -inline String toString(const StringRange & range) -{ - return range.first ? String(range.first, range.second) : String(); -} - -/// Hashes only the values of pointers in StringRange. Is used with StringRangePointersEqualTo comparator. -struct StringRangePointersHash -{ - UInt64 operator()(const StringRange & range) const - { - SipHash hash; - hash.update(range.first); - hash.update(range.second); - return hash.get64(); - } -}; - -/// Ranges are equal only when they point to the same memory region. -/// It may be used when it's enough to compare substrings by their position in the same string. -struct StringRangePointersEqualTo -{ - constexpr bool operator()(const StringRange &lhs, const StringRange &rhs) const - { - return std::tie(lhs.first, lhs.second) == std::tie(rhs.first, rhs.second); - } -}; - -} - diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 81df330ffb5..26aa3555c2b 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -1534,15 +1534,23 @@ static ColumnWithTypeAndName readColumnWithDateData( for (size_t i = 0; i < orc_int_column->numElements; ++i) { - Int32 days_num = static_cast(orc_int_column->data[i]); - if (check_date_range && (days_num > DATE_LUT_MAX_EXTEND_DAY_NUM || days_num < -DAYNUM_OFFSET_EPOCH)) - throw Exception( - ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, - "Input value {} of a column \"{}\" exceeds the range of type Date32", - days_num, - column_name); + if (!orc_int_column->hasNulls || orc_int_column->notNull[i]) + { + Int32 days_num = static_cast(orc_int_column->data[i]); + if (check_date_range && (days_num > DATE_LUT_MAX_EXTEND_DAY_NUM || days_num < -DAYNUM_OFFSET_EPOCH)) + throw Exception( + ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, + "Input value {} of a column \"{}\" exceeds the range of type Date32", + days_num, + column_name); - column_data.push_back(days_num); + column_data.push_back(days_num); + } + else + { + /// ORC library doesn't guarantee that orc_int_column->data[i] is initialized to zero when orc_int_column->notNull[i] is false since https://github.com/ClickHouse/ClickHouse/pull/69473 + column_data.push_back(0); + } } return {std::move(internal_column), internal_type, column_name}; diff --git a/src/Processors/LimitTransform.cpp b/src/Processors/LimitTransform.cpp index 5e24062d67a..a531dd599b2 100644 --- a/src/Processors/LimitTransform.cpp +++ b/src/Processors/LimitTransform.cpp @@ -317,8 +317,9 @@ void LimitTransform::splitChunk(PortsData & data) length = offset + limit - (rows_read - num_rows) - start; } - /// check if other rows in current block equals to last one in limit - if (with_ties && length) + /// Check if other rows in current block equals to last one in limit + /// when rows read >= offset + limit. + if (with_ties && offset + limit <= rows_read && length) { UInt64 current_row_num = start + length; previous_row_chunk = makeChunkWithPreviousRow(data.current_chunk, current_row_num - 1); diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp index fb3ed7f80fc..1832cc2ad42 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp @@ -6,12 +6,23 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool query_plan_merge_filters; +} + BuildQueryPipelineSettings BuildQueryPipelineSettings::fromContext(ContextPtr from) { + const auto & query_settings = from->getSettingsRef(); BuildQueryPipelineSettings settings; - settings.actions_settings = ExpressionActionsSettings::fromSettings(from->getSettingsRef(), CompileExpressions::yes); + settings.actions_settings = ExpressionActionsSettings::fromSettings(query_settings, CompileExpressions::yes); settings.process_list_element = from->getProcessListElement(); settings.progress_callback = from->getProgressCallback(); + + /// Setting query_plan_merge_filters is enabled by default. + /// But it can brake short-circuit without splitting filter step into smaller steps. + /// So, enable and disable this optimizations together. + settings.enable_multiple_filters_transforms_for_and_chain = query_settings[Setting::query_plan_merge_filters]; return settings; } diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h index d99f9a7d1f1..6219e37db58 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h @@ -17,6 +17,8 @@ using TemporaryFileLookupPtr = std::shared_ptr; struct BuildQueryPipelineSettings { + bool enable_multiple_filters_transforms_for_and_chain = true; + ExpressionActionsSettings actions_settings; QueryStatusPtr process_list_element; ProgressCallback progress_callback = nullptr; diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 862e03d74f2..af9e3f0c515 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -5,6 +5,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB { @@ -24,6 +29,92 @@ static ITransformingStep::Traits getTraits() }; } +static bool isTrivialSubtree(const ActionsDAG::Node * node) +{ + while (node->type == ActionsDAG::ActionType::ALIAS) + node = node->children.at(0); + + return node->type != ActionsDAG::ActionType::FUNCTION && node->type != ActionsDAG::ActionType::ARRAY_JOIN; +} + +struct ActionsAndName +{ + ActionsDAG dag; + std::string name; +}; + +static ActionsAndName splitSingleAndFilter(ActionsDAG & dag, const ActionsDAG::Node * filter_node) +{ + auto split_result = dag.split({filter_node}, true); + dag = std::move(split_result.second); + + const auto * split_filter_node = split_result.split_nodes_mapping[filter_node]; + auto filter_type = removeLowCardinality(split_filter_node->result_type); + if (!filter_type->onlyNull() && !isUInt8(removeNullable(filter_type))) + { + DataTypePtr cast_type = std::make_shared(); + if (filter_type->isNullable()) + cast_type = std::make_shared(std::move(cast_type)); + + split_filter_node = &split_result.first.addCast(*split_filter_node, cast_type, {}); + } + + split_result.first.getOutputs().emplace(split_result.first.getOutputs().begin(), split_filter_node); + auto name = split_filter_node->result_name; + return ActionsAndName{std::move(split_result.first), std::move(name)}; +} + +/// Try to split the left most AND atom to a separate DAG. +static std::optional trySplitSingleAndFilter(ActionsDAG & dag, const std::string & filter_name) +{ + const auto * filter = &dag.findInOutputs(filter_name); + while (filter->type == ActionsDAG::ActionType::ALIAS) + filter = filter->children.at(0); + + if (filter->type != ActionsDAG::ActionType::FUNCTION || filter->function_base->getName() != "and") + return {}; + + const ActionsDAG::Node * condition_to_split = nullptr; + std::stack nodes; + nodes.push(filter); + while (!nodes.empty()) + { + const auto * node = nodes.top(); + nodes.pop(); + + if (node->type == ActionsDAG::ActionType::FUNCTION && node->function_base->getName() == "and") + { + /// The order is important. We should take the left-most atom, so put conditions on stack in reverse order. + for (const auto * child : node->children | std::ranges::views::reverse) + nodes.push(child); + + continue; + } + + if (isTrivialSubtree(node)) + continue; + + /// Do not split subtree if it's the last non-trivial one. + /// So, split the first found condition only when there is a another one found. + if (condition_to_split) + return splitSingleAndFilter(dag, condition_to_split); + + condition_to_split = node; + } + + return {}; +} + +std::vector splitAndChainIntoMultipleFilters(ActionsDAG & dag, const std::string & filter_name) +{ + std::vector res; + + while (auto condition = trySplitSingleAndFilter(dag, filter_name)) + res.push_back(std::move(*condition)); + + return res; +} + FilterStep::FilterStep( const Header & input_header_, ActionsDAG actions_dag_, @@ -50,6 +141,23 @@ FilterStep::FilterStep( void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) { + std::vector and_atoms; + + /// Splitting AND filter condition to steps under the setting, which is enabled with merge_filters optimization. + /// This is needed to support short-circuit properly. + if (settings.enable_multiple_filters_transforms_for_and_chain && !actions_dag.hasStatefulFunctions()) + and_atoms = splitAndChainIntoMultipleFilters(actions_dag, filter_column_name); + + for (auto & and_atom : and_atoms) + { + auto expression = std::make_shared(std::move(and_atom.dag), settings.getActionsSettings()); + pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) + { + bool on_totals = stream_type == QueryPipelineBuilder::StreamType::Totals; + return std::make_shared(header, expression, and_atom.name, true, on_totals); + }); + } + auto expression = std::make_shared(std::move(actions_dag), settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) @@ -76,18 +184,45 @@ void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ void FilterStep::describeActions(FormatSettings & settings) const { String prefix(settings.offset, settings.indent_char); + + auto cloned_dag = actions_dag.clone(); + + std::vector and_atoms; + if (!actions_dag.hasStatefulFunctions()) + and_atoms = splitAndChainIntoMultipleFilters(cloned_dag, filter_column_name); + + for (auto & and_atom : and_atoms) + { + auto expression = std::make_shared(std::move(and_atom.dag)); + settings.out << prefix << "AND column: " << and_atom.name << '\n'; + expression->describeActions(settings.out, prefix); + } + settings.out << prefix << "Filter column: " << filter_column_name; if (remove_filter_column) settings.out << " (removed)"; settings.out << '\n'; - auto expression = std::make_shared(actions_dag.clone()); + auto expression = std::make_shared(std::move(cloned_dag)); expression->describeActions(settings.out, prefix); } void FilterStep::describeActions(JSONBuilder::JSONMap & map) const { + auto cloned_dag = actions_dag.clone(); + + std::vector and_atoms; + if (!actions_dag.hasStatefulFunctions()) + and_atoms = splitAndChainIntoMultipleFilters(cloned_dag, filter_column_name); + + for (auto & and_atom : and_atoms) + { + auto expression = std::make_shared(std::move(and_atom.dag)); + map.add("AND column", and_atom.name); + map.add("Expression", expression->toTree()); + } + map.add("Filter Column", filter_column_name); map.add("Removes Filter", remove_filter_column); diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 6232fc7f54f..55a9d18f063 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -32,7 +32,7 @@ struct QueryPlanOptimizationSettings bool merge_expressions = true; /// If merge-filters optimization is enabled. - bool merge_filters = false; + bool merge_filters = true; /// If filter push down optimization is enabled. bool filter_push_down = true; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 26936a19a20..626e43898e4 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -176,6 +176,7 @@ namespace Setting extern const SettingsBool use_skip_indexes; extern const SettingsBool use_skip_indexes_if_final; extern const SettingsBool use_uncompressed_cache; + extern const SettingsBool query_plan_merge_filters; extern const SettingsUInt64 merge_tree_min_read_task_size; extern const SettingsBool read_in_order_use_virtual_row; } @@ -208,6 +209,7 @@ static MergeTreeReaderSettings getMergeTreeReaderSettings( .use_asynchronous_read_from_pool = settings[Setting::allow_asynchronous_read_from_io_pool_for_merge_tree] && (settings[Setting::max_streams_to_max_threads_ratio] > 1 || settings[Setting::max_streams_for_merge_tree_reading] > 1), .enable_multiple_prewhere_read_steps = settings[Setting::enable_multiple_prewhere_read_steps], + .force_short_circuit_execution = settings[Setting::query_plan_merge_filters] }; } diff --git a/src/Processors/Transforms/AddingDefaultsTransform.cpp b/src/Processors/Transforms/AddingDefaultsTransform.cpp index a15ea9d67cb..122691e4ea8 100644 --- a/src/Processors/Transforms/AddingDefaultsTransform.cpp +++ b/src/Processors/Transforms/AddingDefaultsTransform.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -184,7 +185,7 @@ void AddingDefaultsTransform::transform(Chunk & chunk) std::unordered_map mixed_columns; - for (const ColumnWithTypeAndName & column_def : evaluate_block) + for (auto & column_def : evaluate_block) { const String & column_name = column_def.name; @@ -199,6 +200,9 @@ void AddingDefaultsTransform::transform(Chunk & chunk) if (!defaults_mask.empty()) { + column_read.column = recursiveRemoveSparse(column_read.column); + column_def.column = recursiveRemoveSparse(column_def.column); + /// TODO: FixedString if (isColumnedAsNumber(column_read.type) || isDecimal(column_read.type)) { diff --git a/src/Storages/IStorageCluster.h b/src/Storages/IStorageCluster.h index 893cf222556..d000e24562f 100644 --- a/src/Storages/IStorageCluster.h +++ b/src/Storages/IStorageCluster.h @@ -10,8 +10,8 @@ namespace DB /** - * Base cluster for Storages used in table functions like s3Cluster and hdfsCluster - * Needed for code simplification around parallel_distributed_insert_select + * Base cluster for Storages used in table functions like s3Cluster and hdfsCluster. + * Necessary for code simplification around parallel_distributed_insert_select. */ class IStorageCluster : public IStorage { diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 51c445945e6..ea01a0ed0f9 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -2263,18 +2263,18 @@ void IMergeTreeDataPart::checkConsistencyWithProjections(bool require_part_metad proj_part->checkConsistency(require_part_metadata); } -void IMergeTreeDataPart::calculateColumnsAndSecondaryIndicesSizesOnDisk() +void IMergeTreeDataPart::calculateColumnsAndSecondaryIndicesSizesOnDisk(std::optional columns_sample) { - calculateColumnsSizesOnDisk(); + calculateColumnsSizesOnDisk(columns_sample); calculateSecondaryIndicesSizesOnDisk(); } -void IMergeTreeDataPart::calculateColumnsSizesOnDisk() +void IMergeTreeDataPart::calculateColumnsSizesOnDisk(std::optional columns_sample) { if (getColumns().empty() || checksums.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot calculate columns sizes when columns or checksums are not initialized"); - calculateEachColumnSizes(columns_sizes, total_columns_size); + calculateEachColumnSizes(columns_sizes, total_columns_size, columns_sample); } void IMergeTreeDataPart::calculateSecondaryIndicesSizesOnDisk() @@ -2521,22 +2521,24 @@ ColumnPtr IMergeTreeDataPart::getColumnSample(const NameAndTypePair & column) co StorageMetadataPtr metadata_ptr = storage.getInMemoryMetadataPtr(); StorageSnapshotPtr storage_snapshot_ptr = std::make_shared(storage, metadata_ptr); + MergeTreeReaderSettings settings; + settings.can_read_part_without_marks = true; MergeTreeReaderPtr reader = getReader( cols, storage_snapshot_ptr, - MarkRanges{MarkRange(0, 1)}, + MarkRanges{MarkRange(0, total_mark)}, /*virtual_fields=*/ {}, /*uncompressed_cache=*/{}, storage.getContext()->getMarkCache().get(), std::make_shared(), - MergeTreeReaderSettings{}, + settings, ValueSizeMap{}, ReadBufferFromFileBase::ProfileCallback{}); Columns result; result.resize(1); - reader->readRows(0, 1, false, 0, result); + reader->readRows(0, total_mark, false, 0, result); return result[0]; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 55f1265318c..24625edf154 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -428,7 +428,7 @@ public: bool shallParticipateInMerges(const StoragePolicyPtr & storage_policy) const; /// Calculate column and secondary indices sizes on disk. - void calculateColumnsAndSecondaryIndicesSizesOnDisk(); + void calculateColumnsAndSecondaryIndicesSizesOnDisk(std::optional columns_sample = std::nullopt); std::optional getRelativePathForPrefix(const String & prefix, bool detached = false, bool broken = false) const; @@ -633,7 +633,7 @@ protected: /// Fill each_columns_size and total_size with sizes from columns files on /// disk using columns and checksums. - virtual void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const = 0; + virtual void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size, std::optional columns_sample) const = 0; std::optional getRelativePathForDetachedPart(const String & prefix, bool broken) const; @@ -715,7 +715,7 @@ private: void loadPartitionAndMinMaxIndex(); - void calculateColumnsSizesOnDisk(); + void calculateColumnsSizesOnDisk(std::optional columns_sample = std::nullopt); void calculateSecondaryIndicesSizesOnDisk(); diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index d1c76505d7c..8923f6a59ca 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -54,6 +54,8 @@ public: const MergeTreeIndexGranularity & getIndexGranularity() const { return index_granularity; } + virtual Block getColumnsSample() const = 0; + protected: SerializationPtr getSerialization(const String & column_name) const; diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 7ba358d2d35..4a7e02a7a51 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -330,7 +330,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( auto prewhere_actions = MergeTreeSelectProcessor::getPrewhereActions( prewhere_info, actions_settings, - reader_settings.enable_multiple_prewhere_read_steps); + reader_settings.enable_multiple_prewhere_read_steps, reader_settings.force_short_circuit_execution); for (const auto & step : prewhere_actions.steps) add_step(*step); diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 14c2da82de1..8856f467b90 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -80,7 +80,7 @@ MergeTreeDataPartWriterPtr createMergeTreeDataPartCompactWriter( } -void MergeTreeDataPartCompact::calculateEachColumnSizes(ColumnSizeByName & /*each_columns_size*/, ColumnSize & total_size) const +void MergeTreeDataPartCompact::calculateEachColumnSizes(ColumnSizeByName & /*each_columns_size*/, ColumnSize & total_size, std::optional /*columns_sample*/) const { auto bin_checksum = checksums.files.find(DATA_FILE_NAME_WITH_EXTENSION); if (bin_checksum != checksums.files.end()) diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 8e279571578..c394de0d7c1 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -70,7 +70,7 @@ private: void loadIndexGranularity() override; /// Compact parts don't support per column size, only total size - void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const override; + void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size, std::optional columns_sample) const override; }; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index c515d645253..39f96ba06ad 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -82,7 +82,7 @@ MergeTreeDataPartWriterPtr createMergeTreeDataPartWideWriter( /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( - const NameAndTypePair & column, std::unordered_set * processed_substreams) const + const NameAndTypePair & column, std::unordered_set * processed_substreams, std::optional columns_sample) const { ColumnSize size; if (checksums.empty()) @@ -108,7 +108,7 @@ ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( auto mrk_checksum = checksums.files.find(*stream_name + getMarksFileExtension()); if (mrk_checksum != checksums.files.end()) size.marks += mrk_checksum->second.file_size; - }); + }, column.type, columns_sample && columns_sample->has(column.name) ? columns_sample->getByName(column.name).column : getColumnSample(column)); return size; } @@ -374,12 +374,12 @@ std::optional MergeTreeDataPartWide::getFileNameForColumn(const NameAndT return filename; } -void MergeTreeDataPartWide::calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const +void MergeTreeDataPartWide::calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size, std::optional columns_sample) const { std::unordered_set processed_substreams; for (const auto & column : columns) { - ColumnSize size = getColumnSizeImpl(column, &processed_substreams); + ColumnSize size = getColumnSizeImpl(column, &processed_substreams, columns_sample); each_columns_size[column.name] = size; total_size.add(size); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index 022a5fb746c..a6d4897ed87 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -64,9 +64,9 @@ private: /// Loads marks index granularity into memory void loadIndexGranularity() override; - ColumnSize getColumnSizeImpl(const NameAndTypePair & column, std::unordered_set * processed_substreams) const; + ColumnSize getColumnSizeImpl(const NameAndTypePair & column, std::unordered_set * processed_substreams, std::optional columns_sample) const; - void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const override; + void calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size, std::optional columns_sample) const override; }; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h index 49d654c15e1..b22d58ba51e 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h @@ -123,6 +123,8 @@ public: written_offset_columns = written_offset_columns_; } + Block getColumnsSample() const override { return block_sample; } + protected: /// Count index_granularity for block and store in `index_granularity` size_t computeIndexGranularity(const Block & block) const; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 1b3c58000e7..c4ca545ca90 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1045,7 +1045,6 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( MarkRanges res; size_t marks_count = part->index_granularity.getMarksCount(); - const auto & index = part->getIndex(); if (marks_count == 0) return res; @@ -1073,14 +1072,19 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( auto index_columns = std::make_shared(); const auto & key_indices = key_condition.getKeyIndices(); DataTypes key_types; - for (size_t i : key_indices) + if (!key_indices.empty()) { - if (i < index->size()) - index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]); - else - index_columns->emplace_back(); /// The column of the primary key was not loaded in memory - we'll skip it. + const auto & index = part->getIndex(); - key_types.emplace_back(primary_key.data_types[i]); + for (size_t i : key_indices) + { + if (i < index->size()) + index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]); + else + index_columns->emplace_back(); /// The column of the primary key was not loaded in memory - we'll skip it. + + key_types.emplace_back(primary_key.data_types[i]); + } } /// If there are no monotonic functions, there is no need to save block reference. diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.h b/src/Storages/MergeTree/MergeTreeIOSettings.h index 4d1d2533729..7506c726bc4 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -45,6 +45,8 @@ struct MergeTreeReaderSettings bool use_asynchronous_read_from_pool = false; /// If PREWHERE has multiple conditions combined with AND, execute them in separate read/filtering steps. bool enable_multiple_prewhere_read_steps = false; + /// In case of multiple prewhere steps, execute filtering earlier to support short-circuit properly. + bool force_short_circuit_execution = false; /// If true, try to lower size of read buffer according to granule size and compressed block size. bool adjust_read_buffer_size = true; /// If true, it's allowed to read the whole part without reading marks. diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 77231d8d392..885bd1ded8c 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -172,7 +172,7 @@ size_t MergeTreeReaderWide::readRows( throw; } - if (column->empty()) + if (column->empty() && max_rows_to_read > 0) res_columns[pos] = nullptr; } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 867040531e4..426e95fd95d 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -91,7 +91,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( , algorithm(std::move(algorithm_)) , prewhere_info(prewhere_info_) , actions_settings(actions_settings_) - , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps)) + , prewhere_actions(getPrewhereActions(prewhere_info, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps, reader_settings_.force_short_circuit_execution)) , reader_settings(reader_settings_) , result_header(transformHeader(pool->getHeader(), prewhere_info)) { @@ -124,9 +124,9 @@ String MergeTreeSelectProcessor::getName() const return fmt::format("MergeTreeSelect(pool: {}, algorithm: {})", pool->getName(), algorithm->getName()); } -bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere); +bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, bool force_short_circuit_execution); -PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps) +PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, bool force_short_circuit_execution) { PrewhereExprInfo prewhere_actions; if (prewhere_info) @@ -147,7 +147,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr } if (!enable_multiple_prewhere_read_steps || - !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions)) + !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution)) { PrewhereExprStep prewhere_step { diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 33069a78e33..32a761cefb7 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -73,7 +73,8 @@ public: static PrewhereExprInfo getPrewhereActions( PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, - bool enable_multiple_prewhere_read_steps); + bool enable_multiple_prewhere_read_steps, + bool force_short_circuit_execution); void addPartLevelToChunk(bool add_part_level_) { add_part_level = add_part_level_; } diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index 9c82817e8cb..1cc4006a285 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -57,9 +58,9 @@ struct DAGNodeRef const ActionsDAG::Node * node; }; -/// Result name -> DAGNodeRef -using OriginalToNewNodeMap = std::unordered_map; -using NodeNameToLastUsedStepMap = std::unordered_map; +/// ResultNode -> DAGNodeRef +using OriginalToNewNodeMap = std::unordered_map; +using NodeNameToLastUsedStepMap = std::unordered_map; /// Clones the part of original DAG responsible for computing the original_dag_node and adds it to the new DAG. const ActionsDAG::Node & addClonedDAGToDAG( @@ -69,25 +70,28 @@ const ActionsDAG::Node & addClonedDAGToDAG( OriginalToNewNodeMap & node_remap, NodeNameToLastUsedStepMap & node_to_step_map) { - const String & node_name = original_dag_node->result_name; /// Look for the node in the map of already known nodes - if (node_remap.contains(node_name)) + if (node_remap.contains(original_dag_node)) { /// If the node is already in the new DAG, return it - const auto & node_ref = node_remap.at(node_name); + const auto & node_ref = node_remap.at(original_dag_node); if (node_ref.dag == new_dag.get()) return *node_ref.node; /// If the node is known from the previous steps, add it as an input, except for constants if (original_dag_node->type != ActionsDAG::ActionType::COLUMN) { - node_ref.dag->addOrReplaceInOutputs(*node_ref.node); + /// If the node was found in node_remap, it was not added to outputs yet. + /// The only exception is the filter node, which is always the first one. + if (node_ref.dag->getOutputs().at(0) != node_ref.node) + node_ref.dag->getOutputs().push_back(node_ref.node); + const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type); - node_remap[node_name] = {new_dag.get(), &new_node}; /// TODO: here we update the node reference. Is it always correct? + node_remap[original_dag_node] = {new_dag.get(), &new_node}; /// Remember the index of the last step which reuses this node. /// We cannot remove this node from the outputs before that step. - node_to_step_map[node_name] = step; + node_to_step_map[original_dag_node] = step; return new_node; } } @@ -96,7 +100,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( if (original_dag_node->type == ActionsDAG::ActionType::INPUT) { const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -105,7 +109,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( { const auto & new_node = new_dag->addColumn( ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name)); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -113,7 +117,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( { const auto & alias_child = addClonedDAGToDAG(step, original_dag_node->children[0], new_dag, node_remap, node_to_step_map); const auto & new_node = new_dag->addAlias(alias_child, original_dag_node->result_name); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -128,7 +132,7 @@ const ActionsDAG::Node & addClonedDAGToDAG( } const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name); - node_remap[node_name] = {new_dag.get(), &new_node}; + node_remap[original_dag_node] = {new_dag.get(), &new_node}; return new_node; } @@ -138,11 +142,9 @@ const ActionsDAG::Node & addClonedDAGToDAG( const ActionsDAG::Node & addFunction( const ActionsDAGPtr & new_dag, const FunctionOverloadResolverPtr & function, - ActionsDAG::NodeRawConstPtrs children, - OriginalToNewNodeMap & node_remap) + ActionsDAG::NodeRawConstPtrs children) { const auto & new_node = new_dag->addFunction(function, children, ""); - node_remap[new_node.result_name] = {new_dag.get(), &new_node}; return new_node; } @@ -152,14 +154,12 @@ const ActionsDAG::Node & addFunction( const ActionsDAG::Node & addCast( const ActionsDAGPtr & dag, const ActionsDAG::Node & node_to_cast, - const DataTypePtr & to_type, - OriginalToNewNodeMap & node_remap) + const DataTypePtr & to_type) { if (!node_to_cast.result_type->equals(*to_type)) return node_to_cast; const auto & new_node = dag->addCast(node_to_cast, to_type, {}); - node_remap[new_node.result_name] = {dag.get(), &new_node}; return new_node; } @@ -169,8 +169,7 @@ const ActionsDAG::Node & addCast( /// 2. makes sure that the result contains only 0 or 1 values even if the source column contains non-boolean values. const ActionsDAG::Node & addAndTrue( const ActionsDAGPtr & dag, - const ActionsDAG::Node & filter_node_to_normalize, - OriginalToNewNodeMap & node_remap) + const ActionsDAG::Node & filter_node_to_normalize) { Field const_true_value(true); @@ -181,7 +180,7 @@ const ActionsDAG::Node & addAndTrue( const auto * const_true_node = &dag->addColumn(std::move(const_true_column)); ActionsDAG::NodeRawConstPtrs children = {&filter_node_to_normalize, const_true_node}; FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - return addFunction(dag, func_builder_and, children, node_remap); + return addFunction(dag, func_builder_and, children); } } @@ -206,7 +205,11 @@ const ActionsDAG::Node & addAndTrue( /// 6. Find all outputs of the original DAG /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 -bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere) +bool tryBuildPrewhereSteps( + PrewhereInfoPtr prewhere_info, + const ExpressionActionsSettings & actions_settings, + PrewhereExprInfo & prewhere, + bool force_short_circuit_execution) { if (!prewhere_info) return true; @@ -243,7 +246,10 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction struct Step { ActionsDAGPtr actions; - String column_name; + /// Original condition, in case if we have only one condition, and it was not casted + const ActionsDAG::Node * original_node; + /// Result condition node + const ActionsDAG::Node * result_node; }; std::vector steps; @@ -254,7 +260,8 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { const auto & condition_group = condition_groups[step_index]; ActionsDAGPtr step_dag = std::make_unique(); - String result_name; + const ActionsDAG::Node * original_node = nullptr; + const ActionsDAG::Node * result_node; std::vector new_condition_nodes; for (const auto * node : condition_group) @@ -267,48 +274,37 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { /// Add AND function to combine the conditions FunctionOverloadResolverPtr func_builder_and = std::make_unique(std::make_shared()); - const auto & and_function_node = addFunction(step_dag, func_builder_and, new_condition_nodes, node_remap); - step_dag->addOrReplaceInOutputs(and_function_node); - result_name = and_function_node.result_name; + const auto & and_function_node = addFunction(step_dag, func_builder_and, new_condition_nodes); + result_node = &and_function_node; } else { - const auto & result_node = *new_condition_nodes.front(); + result_node = new_condition_nodes.front(); /// Check if explicit cast is needed for the condition to serve as a filter. - const auto result_type_name = result_node.result_type->getName(); - if (result_type_name == "UInt8" || - result_type_name == "Nullable(UInt8)" || - result_type_name == "LowCardinality(UInt8)" || - result_type_name == "LowCardinality(Nullable(UInt8))") - { - /// No need to cast - step_dag->addOrReplaceInOutputs(result_node); - result_name = result_node.result_name; - } - else + if (!isUInt8(removeNullable(removeLowCardinality(result_node->result_type)))) { /// Build "condition AND True" expression to "cast" the condition to UInt8 or Nullable(UInt8) depending on its type. - const auto & cast_node = addAndTrue(step_dag, result_node, node_remap); - step_dag->addOrReplaceInOutputs(cast_node); - result_name = cast_node.result_name; + result_node = &addAndTrue(step_dag, *result_node); } } - steps.push_back({std::move(step_dag), result_name}); + step_dag->getOutputs().insert(step_dag->getOutputs().begin(), result_node); + steps.push_back({std::move(step_dag), original_node, result_node}); } /// 6. Find all outputs of the original DAG auto original_outputs = prewhere_info->prewhere_actions.getOutputs(); + steps.back().actions->getOutputs().clear(); /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4 - NameSet all_output_names; + std::unordered_set all_outputs; for (const auto * output : original_outputs) { - all_output_names.insert(output->result_name); - if (node_remap.contains(output->result_name)) + all_outputs.insert(output); + if (node_remap.contains(output)) { - const auto & new_node_info = node_remap[output->result_name]; - new_node_info.dag->addOrReplaceInOutputs(*new_node_info.node); + const auto & new_node_info = node_remap[output]; + new_node_info.dag->getOutputs().push_back(new_node_info.node); } else if (output->result_name == prewhere_info->prewhere_column_name) { @@ -319,20 +315,21 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction /// 1. AND the last condition with constant True. This is needed to make sure that in the last step filter has UInt8 type /// but contains values other than 0 and 1 (e.g. if it is (number%5) it contains 2,3,4) /// 2. CAST the result to the exact type of the PREWHERE column from the original DAG - const auto & last_step_result_node_info = node_remap[steps.back().column_name]; auto & last_step_dag = steps.back().actions; + auto & last_step_result_node = steps.back().result_node; /// Build AND(last_step_result_node, true) - const auto & and_node = addAndTrue(last_step_dag, *last_step_result_node_info.node, node_remap); + const auto & and_node = addAndTrue(last_step_dag, *last_step_result_node); /// Build CAST(and_node, type of PREWHERE column) - const auto & cast_node = addCast(last_step_dag, and_node, output->result_type, node_remap); + const auto & cast_node = addCast(last_step_dag, and_node, output->result_type); /// Add alias for the result with the name of the PREWHERE column const auto & prewhere_result_node = last_step_dag->addAlias(cast_node, output->result_name); - last_step_dag->addOrReplaceInOutputs(prewhere_result_node); + last_step_dag->getOutputs().push_back(&prewhere_result_node); + steps.back().result_node = &prewhere_result_node; } else { const auto & node_in_new_dag = addClonedDAGToDAG(steps.size() - 1, output, steps.back().actions, node_remap, node_to_step); - steps.back().actions->addOrReplaceInOutputs(node_in_new_dag); + steps.back().actions->getOutputs().push_back(&node_in_new_dag); } } @@ -345,17 +342,18 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction { .type = PrewhereExprStep::Filter, .actions = std::make_shared(std::move(*step.actions), actions_settings), - .filter_column_name = step.column_name, + .filter_column_name = step.result_node->result_name, /// Don't remove if it's in the list of original outputs .remove_filter_column = - !all_output_names.contains(step.column_name) && node_to_step[step.column_name] <= step_index, - .need_filter = false, + step.original_node && !all_outputs.contains(step.original_node) && node_to_step[step.original_node] <= step_index, + .need_filter = force_short_circuit_execution, .perform_alter_conversions = true, }; prewhere.steps.push_back(std::make_shared(std::move(new_step))); } + prewhere.steps.back()->remove_filter_column = prewhere_info->remove_prewhere_column; prewhere.steps.back()->need_filter = prewhere_info->need_filter; } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 39096718b5c..14a521ce429 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -209,7 +209,7 @@ MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( new_part->index_granularity = writer->getIndexGranularity(); /// Just in case new_part->index_granularity.shrinkToFitInMemory(); - new_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); + new_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(writer->getColumnsSample()); /// In mutation, existing_rows_count is already calculated in PartMergerWriter /// In merge situation, lightweight deleted rows was physically deleted, existing_rows_count equals rows_count diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index f3ae6e77ac3..c0e25a54bf3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -619,8 +619,17 @@ bool ReplicatedMergeTreeSinkImpl::writeExistingPart(MergeTreeData::Mutabl String block_id = deduplicate ? fmt::format("{}_{}", part->info.partition_id, part->checksums.getTotalChecksumHex()) : ""; bool deduplicated = commitPart(zookeeper, part, block_id, replicas_num).second; + int error = 0; /// Set a special error code if the block is duplicate - int error = (deduplicate && deduplicated) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; + /// And remove attaching_ prefix + if (deduplicate && deduplicated) + { + error = ErrorCodes::INSERT_WAS_DEDUPLICATED; + if (!endsWith(part->getDataPartStorage().getRelativePath(), "detached/attaching_" + part->name + "/")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected relative path for a deduplicated part: {}", part->getDataPartStorage().getRelativePath()); + fs::path new_relative_path = fs::path("detached") / part->getNewName(part->info); + part->renameTo(new_relative_path, false); + } PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, watch.elapsed(), profile_events_scope.getSnapshot()), ExecutionStatus(error)); return deduplicated; } @@ -880,8 +889,9 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: } } - /// Save the current temporary path in case we need to revert the change to retry (ZK connection loss) + /// Save the current temporary path and name in case we need to revert the change to retry (ZK connection loss) or in case part is deduplicated. const String temporary_part_relative_path = part->getDataPartStorage().getPartDirectory(); + const String initial_part_name = part->name; /// Obtain incremental block number and lock it. The lock holds our intention to add the block to the filesystem. /// We remove the lock just after renaming the part. In case of exception, block number will be marked as abandoned. @@ -1050,16 +1060,6 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: zkutil::KeeperMultiException::check(multi_code, ops, responses); } - transaction.rollback(); - - if (!Coordination::isUserError(multi_code)) - throw Exception( - ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR, - "Unexpected ZooKeeper error while adding block {} with ID '{}': {}", - block_number, - toString(block_id), - multi_code); - auto failed_op_idx = zkutil::getFailedOpIndex(multi_code, responses); String failed_op_path = ops[failed_op_idx]->getPath(); @@ -1069,6 +1069,11 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: LOG_INFO(log, "Block with ID {} already exists (it was just appeared) for part {}. Ignore it.", toString(block_id), part->name); + transaction.rollbackPartsToTemporaryState(); + part->is_temp = true; + part->setName(initial_part_name); + part->renameTo(temporary_part_relative_path, false); + if constexpr (async_insert) { retry_context.conflict_block_ids = std::vector({failed_op_path}); @@ -1080,6 +1085,16 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: return CommitRetryContext::DUPLICATED_PART; } + transaction.rollback(); + + if (!Coordination::isUserError(multi_code)) + throw Exception( + ErrorCodes::UNEXPECTED_ZOOKEEPER_ERROR, + "Unexpected ZooKeeper error while adding block {} with ID '{}': {}", + block_number, + toString(block_id), + multi_code); + if (multi_code == Coordination::Error::ZNONODE && failed_op_idx == block_unlock_op_idx) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Insert query (for block {}) was canceled by concurrent ALTER PARTITION or TRUNCATE", diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 563bdc44760..1ccf23ade90 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -517,7 +517,7 @@ std::unique_ptr StorageObjectStorageSource::createReadBu LOG_TRACE(log, "Downloading object of size {} with initial prefetch", object_size); - bool prefer_bigger_buffer_size = impl->isCached(); + bool prefer_bigger_buffer_size = read_settings.filesystem_cache_prefer_bigger_buffer_size && impl->isCached(); size_t buffer_size = prefer_bigger_buffer_size ? std::max(read_settings.remote_fs_buffer_size, DBMS_DEFAULT_BUFFER_SIZE) : read_settings.remote_fs_buffer_size; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index 060f1cd2dd5..47eb57f918d 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -30,8 +30,8 @@ namespace ErrorCodes DECLARE(UInt64, tracked_files_limit, 1000, "For unordered mode. Max set size for tracking processed files in ZooKeeper", 0) \ DECLARE(UInt64, tracked_file_ttl_sec, 0, "Maximum number of seconds to store processed files in ZooKeeper node (store forever by default)", 0) \ DECLARE(UInt64, polling_min_timeout_ms, 1000, "Minimal timeout before next polling", 0) \ - DECLARE(UInt64, polling_max_timeout_ms, 10000, "Maximum timeout before next polling", 0) \ - DECLARE(UInt64, polling_backoff_ms, 1000, "Polling backoff", 0) \ + DECLARE(UInt64, polling_max_timeout_ms, 10 * 60 * 1000, "Maximum timeout before next polling", 0) \ + DECLARE(UInt64, polling_backoff_ms, 30 * 1000, "Polling backoff", 0) \ DECLARE(UInt32, cleanup_interval_min_ms, 60000, "For unordered mode. Polling backoff min for cleanup", 0) \ DECLARE(UInt32, cleanup_interval_max_ms, 60000, "For unordered mode. Polling backoff max for cleanup", 0) \ DECLARE(UInt32, buckets, 0, "Number of buckets for Ordered mode parallel processing", 0) \ diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index ba1a97bc2fb..e702f07208a 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -659,7 +659,7 @@ void ObjectStorageQueueSource::applyActionAfterProcessing(const String & path) { if (files_metadata->getTableMetadata().after_processing == ObjectStorageQueueAction::DELETE) { - object_storage->removeObject(StoredObject(path)); + object_storage->removeObjectIfExists(StoredObject(path)); } } diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 200872a2f4c..6630d5a6c8f 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -747,6 +747,7 @@ ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const /// so let's reconstruct. ObjectStorageQueueSettings settings; const auto & table_metadata = getTableMetadata(); + settings[ObjectStorageQueueSetting::mode] = table_metadata.mode; settings[ObjectStorageQueueSetting::after_processing] = table_metadata.after_processing; settings[ObjectStorageQueueSetting::keeper_path] = zk_path; settings[ObjectStorageQueueSetting::loading_retries] = table_metadata.loading_retries; @@ -764,6 +765,7 @@ ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const settings[ObjectStorageQueueSetting::max_processed_files_before_commit] = commit_settings.max_processed_files_before_commit; settings[ObjectStorageQueueSetting::max_processed_rows_before_commit] = commit_settings.max_processed_rows_before_commit; settings[ObjectStorageQueueSetting::max_processed_bytes_before_commit] = commit_settings.max_processed_bytes_before_commit; + settings[ObjectStorageQueueSetting::max_processing_time_sec_before_commit] = commit_settings.max_processing_time_sec_before_commit; return settings; } diff --git a/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp b/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp index df0f6b8bc5c..b8a3b2911b9 100644 --- a/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp +++ b/src/Storages/TimeSeries/PrometheusRemoteReadProtocol.cpp @@ -245,6 +245,7 @@ namespace table_join->strictness = JoinStrictness::Semi; table_join->on_expression = makeASTFunction("equals", makeASTColumn(data_table_id, TimeSeriesColumnNames::ID), makeASTColumn(tags_table_id, TimeSeriesColumnNames::ID)); + table_join->children.push_back(table_join->on_expression); table->table_join = table_join; auto table_exp = std::make_shared(); diff --git a/src/Storages/Utils.cpp b/src/Storages/Utils.cpp index bd03a96c7cc..72aeb0d158d 100644 --- a/src/Storages/Utils.cpp +++ b/src/Storages/Utils.cpp @@ -1,10 +1,13 @@ +#include #include #include +#include namespace CurrentMetrics { extern const Metric AttachedTable; + extern const Metric AttachedReplicatedTable; extern const Metric AttachedView; extern const Metric AttachedDictionary; } @@ -12,17 +15,20 @@ namespace CurrentMetrics namespace DB { - CurrentMetrics::Metric getAttachedCounterForStorage(const StoragePtr & storage) + std::vector getAttachedCountersForStorage(const StoragePtr & storage) { if (storage->isView()) { - return CurrentMetrics::AttachedView; + return {CurrentMetrics::AttachedView}; } if (storage->isDictionary()) { - return CurrentMetrics::AttachedDictionary; + return {CurrentMetrics::AttachedDictionary}; } - - return CurrentMetrics::AttachedTable; + if (auto * replicated_storage = typeid_cast(storage.get())) + { + return {CurrentMetrics::AttachedTable, CurrentMetrics::AttachedReplicatedTable}; + } + return {CurrentMetrics::AttachedTable}; } } diff --git a/src/Storages/Utils.h b/src/Storages/Utils.h index c86c2a4c341..eb302178485 100644 --- a/src/Storages/Utils.h +++ b/src/Storages/Utils.h @@ -6,5 +6,5 @@ namespace DB { - CurrentMetrics::Metric getAttachedCounterForStorage(const StoragePtr & storage); + std::vector getAttachedCountersForStorage(const StoragePtr & storage); } diff --git a/src/TableFunctions/ITableFunctionCluster.h b/src/TableFunctions/ITableFunctionCluster.h index 744d7139d16..6935ac39e79 100644 --- a/src/TableFunctions/ITableFunctionCluster.h +++ b/src/TableFunctions/ITableFunctionCluster.h @@ -51,7 +51,7 @@ protected: "corresponding table function", getName()); - /// Evaluate only first argument, everything else will be done Base class + /// Evaluate only first argument, everything else will be done by the Base class args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context); /// Cluster name is always the first diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 10431ce038f..eb0fe7c85c3 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -798,10 +798,6 @@ def _upload_build_profile_data( logging.info("Unknown CI logs host, skip uploading build profile data") return - if not pr_info.number == 0: - logging.info("Skipping uploading build profile data for PRs") - return - instance_type = get_instance_type() instance_id = get_instance_id() auth = { @@ -1268,6 +1264,7 @@ def main() -> int: s3, pr_info.number, pr_info.sha, + pr_info.head_ref, job_report.test_results, job_report.additional_files, job_report.check_name or _get_ext_check_name(args.job_name), @@ -1335,6 +1332,7 @@ def main() -> int: s3, pr_info.number, pr_info.sha, + pr_info.head_ref, job_report.test_results, job_report.additional_files, job_report.check_name or _get_ext_check_name(args.job_name), diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py index 99f4778fe8f..32e7f6f6a53 100644 --- a/tests/ci/clickhouse_helper.py +++ b/tests/ci/clickhouse_helper.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional import requests +from env_helper import GITHUB_REPOSITORY from get_robot_token import get_parameter_from_ssm from pr_info import PRInfo from report import TestResults @@ -211,17 +212,13 @@ def prepare_tests_results_for_clickhouse( report_url: str, check_name: str, ) -> List[dict]: - pull_request_url = "https://github.com/ClickHouse/ClickHouse/commits/master" - base_ref = "master" - head_ref = "master" - base_repo = pr_info.repo_full_name - head_repo = pr_info.repo_full_name + base_ref = pr_info.base_ref + base_repo = pr_info.base_name + head_ref = pr_info.head_ref + head_repo = pr_info.head_name + pull_request_url = f"https://github.com/{GITHUB_REPOSITORY}/commits/{head_ref}" if pr_info.number != 0: pull_request_url = pr_info.pr_html_url - base_ref = pr_info.base_ref - base_repo = pr_info.base_name - head_ref = pr_info.head_ref - head_repo = pr_info.head_name common_properties = { "pull_request_number": pr_info.number, diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 28323b6786c..ce9eaf78e75 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -315,7 +315,13 @@ def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str: ) ) return upload_results( - S3Helper(), pr_info.number, pr_info.sha, test_results, [], CI.StatusNames.CI + S3Helper(), + pr_info.number, + pr_info.sha, + pr_info.head_ref, + test_results, + [], + CI.StatusNames.CI, ) diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index c8dbcd10245..c94aff1ab4d 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -250,7 +250,9 @@ def main(): s3_helper = S3Helper() pr_info = PRInfo() - url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME) + url = upload_results( + s3_helper, pr_info.number, pr_info.sha, pr_info.head_ref, test_results, [], NAME + ) print(f"::notice ::Report url: {url}") diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 6c6a88330ea..772cbbf1b02 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -183,7 +183,9 @@ def main(): pr_info = PRInfo() s3_helper = S3Helper() - url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME) + url = upload_results( + s3_helper, pr_info.number, pr_info.sha, pr_info.head_ref, test_results, [], NAME + ) print(f"::notice ::Report url: {url}") diff --git a/tests/ci/official_docker.py b/tests/ci/official_docker.py new file mode 100644 index 00000000000..f62002ee1d2 --- /dev/null +++ b/tests/ci/official_docker.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python +"""Plan: +- Create a PR with regenerated Dockerfiles for each release branch +- Generate `library definition file` as described in +https://github.com/docker-library/official-images/blob/master/README.md#library-definition-files +- Create a PR with it to https://github.com/docker-library/official-images, the file +name will be `library/clickhouse`""" + +import argparse +import logging +from dataclasses import dataclass +from os import getpid +from pathlib import Path +from pprint import pformat +from shlex import quote +from shutil import rmtree +from textwrap import fill +from typing import Dict, Iterable, List, Optional, Set + +from env_helper import GITHUB_REPOSITORY, IS_CI +from git_helper import GIT_PREFIX, Git, git_runner +from version_helper import ( + ClickHouseVersion, + get_supported_versions, + get_tagged_versions, + get_version_from_string, +) + +UBUNTU_NAMES = { + "20.04": "focal", + "22.04": "jammy", +} + +if not IS_CI: + GIT_PREFIX = "git" + +DOCKER_LIBRARY_REPOSITORY = "ClickHouse/docker-library" + +DOCKER_LIBRARY_NAME = {"server": "clickhouse"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="The script to handle tasks for docker-library/official-images", + ) + global_args = argparse.ArgumentParser(add_help=False) + global_args.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="set the script verbosity, could be used multiple", + ) + global_args.add_argument( + "--directory", + type=Path, + default=Path("docker/official"), + help="a relative to the reporitory root directory", + ) + global_args.add_argument( + "--image-type", + choices=["server", "keeper"], + default="server", + help="which image type to process", + ) + global_args.add_argument( + "--commit", + action="store_true", + help="if set, the directory `docker/official` will be staged and committed " + "after generating", + ) + dockerfile_glob = "Dockerfile.*" + global_args.add_argument( + "--dockerfile-glob", + default=dockerfile_glob, + help="a glob to collect Dockerfiles in the server of keeper directory", + ) + subparsers = parser.add_subparsers( + title="commands", dest="command", required=True, help="the command to run" + ) + parser_tree = subparsers.add_parser( + "generate-tree", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + help="generates directory `docker/official`", + parents=[global_args], + ) + parser_tree.add_argument( + "--min-version", + type=get_version_from_string, + default=None, + help="if not set, only currently supported versions will be used", + ) + parser_tree.add_argument( + "--docker-branch", + default="", + help="if set, the branch to get the content of `docker/server` directory. When " + "unset, the content of directories is taken from release tags", + ) + parser_tree.add_argument( + "--build-images", + action="store_true", + help="if set, the image will be built for each Dockerfile '--dockerfile-glob' " + "in the result dirs", + ) + parser_tree.add_argument("--clean", default=True, help=argparse.SUPPRESS) + parser_tree.add_argument( + "--no-clean", + dest="clean", + action="store_false", + default=argparse.SUPPRESS, + help="if set, the directory `docker/official` won't be cleaned " + "before generating", + ) + parser_tree.add_argument( + "--fetch-tags", + action="store_true", + help="if set, the tags will be updated before run", + ) + parser_ldf = subparsers.add_parser( + "generate-ldf", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + help="generate docker library definition file", + parents=[global_args], + ) + parser_ldf.add_argument("--check-changed", default=True, help=argparse.SUPPRESS) + parser_ldf.add_argument( + "--no-check-changed", + dest="check_changed", + action="store_false", + default=argparse.SUPPRESS, + help="if set, the directory `docker/official` won't be checked to be " + "uncommitted", + ) + args = parser.parse_args() + return args + + +def get_versions_greater(minimal: ClickHouseVersion) -> Set[ClickHouseVersion]: + "Get the latest patch version for each major.minor >= minimal" + supported = {} # type: Dict[str, ClickHouseVersion] + versions = get_tagged_versions() + for v in versions: + if v < minimal or not v.is_supported: + continue + txt = f"{v.major}.{v.minor}" + if txt in supported: + supported[txt] = max(v, supported[txt]) + continue + supported[txt] = v + return set(supported.values()) + + +def create_versions_dirs( + versions: Iterable[ClickHouseVersion], directory: Path +) -> Dict[ClickHouseVersion, Path]: + assert directory.is_dir() or not directory.exists() + dirs = {} + for v in versions: + version_dir = directory / str(v) + version_dir.mkdir(parents=True, exist_ok=True) + logging.debug("Directory %s for version %s is created", version_dir, v) + dirs[v] = version_dir + return dirs + + +def generate_docker_directories( + version_dirs: Dict[ClickHouseVersion, Path], + docker_branch: str, + dockerfile_glob: str, + build_images: bool = False, + image_type: str = "server", +) -> None: + arg_version = "ARG VERSION=" + start_filter = "#docker-official-library:off" + stop_filter = "#docker-official-library:on" + for version, directory in version_dirs.items(): + branch = docker_branch or version.describe + docker_source = f"{branch}:docker/{image_type}" + logging.debug( + "Unpack directory content from '%s' to %s", + docker_source, + directory, + ) + # We ignore README* files + git_runner( + f"git archive {docker_source} | tar --exclude='README*' -x -C {directory}" + ) + for df in directory.glob(dockerfile_glob): + original_content = df.read_text().splitlines() + content = [] + filtering = False + for line in original_content: + # Change the ARG VERSION= to a proper version + if line.startswith(arg_version): + logging.debug( + "Found '%s' in line %s:%s, setting to version %s", + arg_version, + df.name, + original_content.index(line), + version, + ) + content.append(f'{arg_version}"{version}"') + continue + # Filter out CI-related part from official docker + if line == start_filter: + filtering = True + continue + if line == stop_filter: + filtering = False + continue + if not filtering: + content.append(line) + + df.write_text("\n".join(content) + "\n") + if build_images: + git_runner( + f"docker build --network=host -t '{DOCKER_LIBRARY_NAME[image_type]}:" + f"{version}{df.suffix}' -f '{df}' --progress plain '{directory}'" + ) + + +def path_is_changed(path: Path) -> bool: + "checks if `path` has uncommitted changes" + logging.info("Checking if the path %s is changed", path) + path_dir = path.parent + return bool(git_runner(f"git -C {path_dir} status --porcelain -- '{path}'")) + + +def get_cmdline(width: int = 80) -> str: + "Returns the cmdline split by words with given maximum width" + cmdline = " ".join( + quote(arg) + for arg in Path(f"/proc/{getpid()}/cmdline") + .read_text(encoding="utf-8") + .split("\x00")[:-1] + ) + if width <= 2: + return cmdline + + return " \\\n".join( + fill( + cmdline, + break_long_words=False, + break_on_hyphens=False, + subsequent_indent=r" ", + width=width - 2, + ).split("\n") + ) + + +def generate_tree(args: argparse.Namespace) -> None: + if args.fetch_tags: + # Fetch all tags to not miss the latest versions + git_runner(f"{GIT_PREFIX} fetch --tags --no-recurse-submodules") + if args.min_version: + versions = get_versions_greater(args.min_version) + else: + versions = get_supported_versions() + logging.info( + "The versions to generate:\n %s", + "\n ".join(v.string for v in sorted(versions)), + ) + directory = Path(git_runner.cwd) / args.directory / args.image_type + if args.clean: + try: + logging.info("Removing directory %s before generating", directory) + rmtree(directory) + except FileNotFoundError: + pass + version_dirs = create_versions_dirs(versions, directory) + generate_docker_directories( + version_dirs, + args.docker_branch, + args.dockerfile_glob, + args.build_images, + args.image_type, + ) + if args.commit and path_is_changed(directory): + logging.info("Staging and committing content of %s", directory) + git_runner(f"git -C {directory} add {directory}") + commit_message = "\n".join( + ( + "Re-/Generated tags for official docker library", + "", + "The changed versions:", + *(f" {v.string}" for v in sorted(versions)), + f"The old images were removed: {args.clean}", + "The directory is generated and committed as following:", + f"{get_cmdline()}", + ) + ) + git_runner(f"{GIT_PREFIX} -C {directory} commit -F -", input=commit_message) + + +@dataclass +class TagAttrs: + """A mutable metadata to preserve between generating tags for different versions""" + + # Only one latest can exist + latest: ClickHouseVersion + # Only one can be a major one (the most fresh per a year) + majors: Dict[int, ClickHouseVersion] + # Only one lts version can exist + lts: Optional[ClickHouseVersion] + + +def ldf_header(git: Git, directory: Path) -> List[str]: + "Generates the header for LDF" + script_path = Path(__file__).relative_to(git.root) + script_sha = git_runner(f"git log -1 --format=format:%H -- {script_path}") + repo = f"https://github.com/{GITHUB_REPOSITORY}" + dl_repo = f"https://github.com/{DOCKER_LIBRARY_REPOSITORY}" + fetch_commit = git_runner( + f"git -C {directory} log -1 --format=format:%H -- {directory}" + ) + dl_branch = git_runner(f"git -C {directory} branch --show-current") + return [ + f"# The file is generated by {repo}/blob/{script_sha}/{script_path}", + "", + "Maintainers: Misha f. Shiryaev (@Felixoid),", + " Max Kainov (@mkaynov),", + " Alexander Sapin (@alesapin)", + f"GitRepo: {dl_repo}.git", + f"GitFetch: refs/heads/{dl_branch}", + f"GitCommit: {fetch_commit}", + ] + + +def ldf_tags(version: ClickHouseVersion, distro: str, tag_attrs: TagAttrs) -> str: + """returns the string 'Tags: coma, separated, tags'""" + tags = [] + # without_distro shows that it's the default tags set, without `-jammy` suffix + without_distro = distro in UBUNTU_NAMES.values() + if version == tag_attrs.latest: + if without_distro: + tags.append("latest") + tags.append(distro) + + # The current version gets the `lts` tag when it's the first met version.is_lts + with_lts = tag_attrs.lts in (None, version) and version.is_lts + if with_lts: + tag_attrs.lts = version + if without_distro: + tags.append("lts") + tags.append(f"lts-{distro}") + + # If the tag `22`, `23`, `24` etc. should be included in the tags + with_major = tag_attrs.majors.get(version.major) in (None, version) + if with_major: + tag_attrs.majors[version.major] = version + if without_distro: + tags.append(f"{version.major}") + tags.append(f"{version.major}-{distro}") + + # Add all normal tags + for tag in ( + f"{version.major}.{version.minor}", + f"{version.major}.{version.minor}.{version.patch}", + f"{version.major}.{version.minor}.{version.patch}.{version.tweak}", + ): + if without_distro: + tags.append(tag) + tags.append(f"{tag}-{distro}") + + return f"Tags: {', '.join(tags)}" + + +def generate_ldf(args: argparse.Namespace) -> None: + """Collect all args.dockerfile_glob files from args.directory and generate the + Library Definition File, read about it in + https://github.com/docker-library/official-images/?tab=readme-ov-file#library-definition-files + """ + directory = Path(git_runner.cwd) / args.directory / args.image_type + versions = sorted([get_version_from_string(d.name) for d in directory.iterdir()]) + assert versions, "There are no directories to generate the LDF" + if args.check_changed: + assert not path_is_changed( + directory + ), f"There are uncommitted changes in {directory}" + git = Git(True) + # Support a few repositories, get the git-root for images directory + dir_git_root = ( + args.directory / git_runner(f"git -C {args.directory} rev-parse --show-cdup") + ).absolute() + lines = ldf_header(git, directory) + tag_attrs = TagAttrs(versions[-1], {}, None) + + # We iterate from the most recent to the oldest version + for version in reversed(versions): + tag_dir = directory / str(version) + for file in tag_dir.glob(args.dockerfile_glob): + lines.append("") + distro = file.suffix[1:] + if distro == "ubuntu": + # replace 'ubuntu' by the release name from UBUNTU_NAMES + with open(file, "r", encoding="utf-8") as fd: + for l in fd: + if l.startswith("FROM ubuntu:"): + ubuntu_version = l.split(":")[-1].strip() + distro = UBUNTU_NAMES[ubuntu_version] + break + lines.append(ldf_tags(version, distro, tag_attrs)) + lines.append("Architectures: amd64, arm64v8") + lines.append(f"Directory: {tag_dir.relative_to(dir_git_root)}") + lines.append(f"File: {file.name}") + + # For the last '\n' in join + lines.append("") + ldf_file = ( + Path(git_runner.cwd) / args.directory / DOCKER_LIBRARY_NAME[args.image_type] + ) + ldf_file.write_text("\n".join(lines)) + logging.info("The content of LDF file:\n%s", "\n".join(lines)) + if args.commit and path_is_changed(ldf_file): + logging.info("Starting committing or %s file", ldf_file) + ldf_dir = ldf_file.parent + git_runner(f"git -C {ldf_dir} add {ldf_file}") + commit_message = ( + f"Re-/Generated docker LDF for {args.image_type} image\n\n" + f"The file is generated and committed as following:\n{get_cmdline()}" + ) + git_runner(f"{GIT_PREFIX} -C {ldf_dir} commit -F -", input=commit_message) + + +def main() -> None: + args = parse_args() + log_levels = [logging.CRITICAL, logging.WARN, logging.INFO, logging.DEBUG] + logging.basicConfig(level=log_levels[min(args.verbose, 3)]) + logging.debug("Arguments are %s", pformat(args.__dict__)) + if args.command == "generate-tree": + generate_tree(args) + elif args.command == "generate-ldf": + generate_ldf(args) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 8ed2f972183..d602a2d752a 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -132,6 +132,12 @@ class PRInfo: ref = github_event.get("ref", "refs/heads/master") if ref and ref.startswith("refs/heads/"): ref = ref[11:] + # Default values + self.base_ref = "" # type: str + self.base_name = "" # type: str + self.head_ref = "" # type: str + self.head_name = "" # type: str + self.number = 0 # type: int # workflow completed event, used for PRs only if "action" in github_event and github_event["action"] == "completed": @@ -146,7 +152,7 @@ class PRInfo: if "pull_request" in github_event: # pull request and other similar events self.event_type = EventType.PULL_REQUEST - self.number = github_event["pull_request"]["number"] # type: int + self.number = github_event["pull_request"]["number"] if pr_event_from_api: try: response = get_gh_api( @@ -172,17 +178,13 @@ class PRInfo: self.pr_html_url = f"{repo_prefix}/pull/{self.number}" # master or backport/xx.x/xxxxx - where the PR will be merged - self.base_ref = github_event["pull_request"]["base"]["ref"] # type: str + self.base_ref = github_event["pull_request"]["base"]["ref"] # ClickHouse/ClickHouse - self.base_name = github_event["pull_request"]["base"]["repo"][ - "full_name" - ] # type: str + self.base_name = github_event["pull_request"]["base"]["repo"]["full_name"] # any_branch-name - the name of working branch name - self.head_ref = github_event["pull_request"]["head"]["ref"] # type: str + self.head_ref = github_event["pull_request"]["head"]["ref"] # UserName/ClickHouse or ClickHouse/ClickHouse - self.head_name = github_event["pull_request"]["head"]["repo"][ - "full_name" - ] # type: str + self.head_name = github_event["pull_request"]["head"]["repo"]["full_name"] self.body = github_event["pull_request"]["body"] self.labels = { label["name"] for label in github_event["pull_request"]["labels"] diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 619ffe937c7..85f4cfd12a0 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -64,6 +64,7 @@ def upload_results( s3_client: S3Helper, pr_number: int, commit_sha: str, + branch_name: str, test_results: TestResults, additional_files: Union[Sequence[Path], Sequence[str]], check_name: str, @@ -80,8 +81,7 @@ def upload_results( process_logs(s3_client, additional_files, s3_path_prefix, test_results) ) - branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master" - branch_name = "master" + branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/{branch_name}" if pr_number != 0: branch_name = f"PR #{pr_number}" branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_number}" diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index bf48f3f53dd..0fce2f920d1 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -164,6 +164,11 @@ class ClickHouseVersion: """our X.3 and X.8 are LTS""" return self.minor % 5 == 3 + @property + def is_supported(self) -> bool: + "we can support only versions with VersionType STABLE or LTS" + return self.description in (VersionType.STABLE, VersionType.LTS) + def get_stable_release_type(self) -> str: if self.is_lts: return VersionType.LTS @@ -365,7 +370,7 @@ def get_supported_versions( versions = list(versions) else: # checks that repo is not shallow in background - versions = get_tagged_versions() + versions = [v for v in get_tagged_versions() if v.is_supported] versions.sort() versions.reverse() for version in versions: diff --git a/tests/docker_scripts/attach_gdb.lib b/tests/docker_scripts/attach_gdb.lib index 4170a19176c..f8a08b5e39d 100644 --- a/tests/docker_scripts/attach_gdb.lib +++ b/tests/docker_scripts/attach_gdb.lib @@ -5,7 +5,8 @@ source /repo/tests/docker_scripts/utils.lib function attach_gdb_to_clickhouse() { - IS_ASAN=$(clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)") + # Use retries to avoid failures due to fault injections + IS_ASAN=$(run_with_retry 5 clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)") if [[ "$IS_ASAN" = "1" ]]; then echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections" diff --git a/tests/integration/compose/docker_compose_rabbitmq.yml b/tests/integration/compose/docker_compose_rabbitmq.yml index 0e5203b925f..7ab5e874110 100644 --- a/tests/integration/compose/docker_compose_rabbitmq.yml +++ b/tests/integration/compose/docker_compose_rabbitmq.yml @@ -16,4 +16,10 @@ services: - /misc/rabbitmq/ca-cert.pem:/etc/rabbitmq/ca-cert.pem - /misc/rabbitmq/server-cert.pem:/etc/rabbitmq/server-cert.pem - /misc/rabbitmq/server-key.pem:/etc/rabbitmq/server-key.pem - - /misc/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins \ No newline at end of file + - /misc/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins + # https://www.rabbitmq.com/docs/monitoring#health-checks + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 10s + retries: 10 + timeout: 2s diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index b24593602ec..a0c2e1d1a70 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -744,11 +744,13 @@ class ClickHouseCluster: # available when with_prometheus == True self.with_prometheus = False self.prometheus_writer_host = "prometheus_writer" + self.prometheus_writer_ip = None self.prometheus_writer_port = 9090 self.prometheus_writer_logs_dir = p.abspath( p.join(self.instances_dir, "prometheus_writer/logs") ) self.prometheus_reader_host = "prometheus_reader" + self.prometheus_reader_ip = None self.prometheus_reader_port = 9091 self.prometheus_reader_logs_dir = p.abspath( p.join(self.instances_dir, "prometheus_reader/logs") @@ -2728,6 +2730,16 @@ class ClickHouseCluster: raise Exception("Can't wait LDAP to start") + def wait_prometheus_to_start(self): + self.prometheus_reader_ip = self.get_instance_ip(self.prometheus_reader_host) + self.prometheus_writer_ip = self.get_instance_ip(self.prometheus_writer_host) + self.wait_for_url( + f"http://{self.prometheus_reader_ip}:{self.prometheus_reader_port}/api/v1/query?query=time()" + ) + self.wait_for_url( + f"http://{self.prometheus_writer_ip}:{self.prometheus_writer_port}/api/v1/query?query=time()" + ) + def start(self): pytest_xdist_logging_to_separate_files.setup() logging.info("Running tests in {}".format(self.base_path)) @@ -3083,12 +3095,23 @@ class ClickHouseCluster: f"http://{self.jdbc_bridge_ip}:{self.jdbc_bridge_port}/ping" ) - if self.with_prometheus: + if self.with_prometheus and self.base_prometheus_cmd: os.makedirs(self.prometheus_writer_logs_dir) os.chmod(self.prometheus_writer_logs_dir, stat.S_IRWXU | stat.S_IRWXO) os.makedirs(self.prometheus_reader_logs_dir) os.chmod(self.prometheus_reader_logs_dir, stat.S_IRWXU | stat.S_IRWXO) + prometheus_start_cmd = self.base_prometheus_cmd + common_opts + + logging.info( + "Trying to create Prometheus instances by command %s", + " ".join(map(str, prometheus_start_cmd)), + ) + run_and_check(prometheus_start_cmd) + self.up_called = True + logging.info("Trying to connect to Prometheus...") + self.wait_prometheus_to_start() + clickhouse_start_cmd = self.base_cmd + ["up", "-d", "--no-recreate"] logging.debug( ( diff --git a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py index f63dc2aef3d..4ad53acc735 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py +++ b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py @@ -251,23 +251,16 @@ def kill_query( if is_initial_query is not None else "" ) + old_time = time.monotonic() node.query( f"KILL QUERY WHERE (query_kind='{query_kind}') AND (query LIKE '%{id}%'){filter_for_is_initial_query} SYNC" ) - node.query("SYSTEM FLUSH LOGS") - duration = ( - int( - node.query( - f"SELECT query_duration_ms FROM system.query_log WHERE query_kind='KillQuery' AND query LIKE '%{id}%' AND type='QueryFinish'" - ) - ) - / 1000 - ) + waited = time.monotonic() - old_time print( - f"{get_node_name(node)}: Cancelled {operation_name} {id} after {duration} seconds" + f"{get_node_name(node)}: Cancelled {operation_name} {id} after {waited} seconds" ) if timeout is not None: - assert duration < timeout + assert waited < timeout # Stops all ZooKeeper servers. @@ -305,7 +298,7 @@ def sleep(seconds): class NoTrashChecker: def __init__(self): self.expect_backups = [] - self.expect_unfinished_backups = [] + self.allow_unfinished_backups = [] self.expect_errors = [] self.allow_errors = [] self.check_zookeeper = True @@ -373,7 +366,7 @@ class NoTrashChecker: if unfinished_backups: print(f"Found unfinished backups: {unfinished_backups}") assert new_backups == set(self.expect_backups) - assert unfinished_backups == set(self.expect_unfinished_backups) + assert unfinished_backups.difference(self.allow_unfinished_backups) == set() all_errors = set() start_time = time.strftime( @@ -641,7 +634,7 @@ def test_long_disconnection_stops_backup(): assert get_status(initiator, backup_id=backup_id) == "CREATING_BACKUP" assert get_num_system_processes(initiator, backup_id=backup_id) >= 1 - no_trash_checker.expect_unfinished_backups = [backup_id] + no_trash_checker.allow_unfinished_backups = [backup_id] no_trash_checker.allow_errors = [ "FAILED_TO_SYNC_BACKUP_OR_RESTORE", "KEEPER_EXCEPTION", @@ -674,7 +667,7 @@ def test_long_disconnection_stops_backup(): # A backup is expected to fail, but it isn't expected to fail too soon. print(f"Backup failed after {time_to_fail} seconds disconnection") assert time_to_fail > 3 - assert time_to_fail < 30 + assert time_to_fail < 35 # A backup must NOT be stopped if Zookeeper is disconnected shorter than `failure_after_host_disconnected_for_seconds`. @@ -695,7 +688,7 @@ def test_short_disconnection_doesnt_stop_backup(): backup_id = random_id() initiator.query( f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {get_backup_name(backup_id)} SETTINGS id='{backup_id}' ASYNC", - settings={"backup_restore_failure_after_host_disconnected_for_seconds": 6}, + settings={"backup_restore_failure_after_host_disconnected_for_seconds": 10}, ) assert get_status(initiator, backup_id=backup_id) == "CREATING_BACKUP" @@ -703,13 +696,13 @@ def test_short_disconnection_doesnt_stop_backup(): # Dropping connection for less than `failure_after_host_disconnected_for_seconds` with PartitionManager() as pm: - random_sleep(3) + random_sleep(4) node_to_drop_zk_connection = random_node() print( f"Dropping connection between {get_node_name(node_to_drop_zk_connection)} and ZooKeeper" ) pm.drop_instance_zk_connections(node_to_drop_zk_connection) - random_sleep(3) + random_sleep(4) print( f"Restoring connection between {get_node_name(node_to_drop_zk_connection)} and ZooKeeper" ) diff --git a/tests/integration/test_deduplicated_attached_part_rename/__init__.py b/tests/integration/test_deduplicated_attached_part_rename/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_deduplicated_attached_part_rename/test.py b/tests/integration/test_deduplicated_attached_part_rename/test.py new file mode 100644 index 00000000000..02fa2c9d4a3 --- /dev/null +++ b/tests/integration/test_deduplicated_attached_part_rename/test.py @@ -0,0 +1,88 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + with_zookeeper=True, + macros={"replica": "node1"}, + stay_alive=True, +) + +database_name = "dedup_attach" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(query): + return ch1.query(database=database_name, sql=query) + + +def test_deduplicated_attached_part_renamed_after_attach(started_cluster): + ch1.query(f"CREATE DATABASE {database_name}") + + q( + "CREATE TABLE dedup (id UInt32) ENGINE=ReplicatedMergeTree('/clickhouse/tables/dedup_attach/dedup/s1', 'r1') ORDER BY id;" + ) + q("INSERT INTO dedup VALUES (1),(2),(3);") + + table_data_path = q( + "SELECT data_paths FROM system.tables WHERE database=currentDatabase() AND table='dedup'" + ).strip("'[]\n") + + ch1.exec_in_container( + [ + "bash", + "-c", + f"cp -r {table_data_path}/all_0_0_0 {table_data_path}/detached/all_0_0_0", + ] + ) + # Part is attached as all_1_1_0 + q("ALTER TABLE dedup ATTACH PART 'all_0_0_0'") + + assert 2 == int( + q( + f"SELECT count() FROM system.parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + + ch1.exec_in_container( + [ + "bash", + "-c", + f"cp -r {table_data_path}/all_1_1_0 {table_data_path}/detached/all_1_1_0", + ] + ) + # Part is deduplicated and not attached + q("ALTER TABLE dedup ATTACH PART 'all_1_1_0'") + + assert 2 == int( + q( + f"SELECT count() FROM system.parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + assert 1 == int( + q( + f"SELECT count() FROM system.detached_parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + # Check that it is not 'attaching_all_1_1_0' + assert ( + "all_1_1_0" + == q( + f"SELECT name FROM system.detached_parts WHERE database='{database_name}' AND table = 'dedup'" + ).strip() + ) + + q("DROP TABLE dedup") + q("SYSTEM DROP REPLICA 'r1' FROM ZKPATH '/clickhouse/tables/dedup_attach/dedup/s1'") + ch1.query(f"DROP DATABASE {database_name}") diff --git a/tests/integration/test_failed_mutations/test.py b/tests/integration/test_failed_mutations/test.py index 5a2bf874da2..24b67ff86e5 100644 --- a/tests/integration/test_failed_mutations/test.py +++ b/tests/integration/test_failed_mutations/test.py @@ -27,6 +27,7 @@ REPLICATED_POSTPONE_MUTATION_LOG = ( POSTPONE_MUTATION_LOG = ( "According to exponential backoff policy, do not perform mutations for the part" ) +FAILING_MUTATION_QUERY = "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT throwIf(1)) SETTINGS allow_nondeterministic_mutations = 1" all_nodes = [node_with_backoff, node_no_backoff] @@ -83,17 +84,13 @@ def test_exponential_backoff_with_merge_tree(started_cluster, node, found_in_log assert not node.contains_in_log(POSTPONE_MUTATION_LOG) # Executing incorrect mutation. - node.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM notexist_table) SETTINGS allow_nondeterministic_mutations=1" - ) + node.query(FAILING_MUTATION_QUERY) check_logs() node.query("KILL MUTATION WHERE table='test_mutations'") # Check that after kill new parts mutations are postponing. - node.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM notexist_table) SETTINGS allow_nondeterministic_mutations=1" - ) + node.query(FAILING_MUTATION_QUERY) check_logs() @@ -101,9 +98,7 @@ def test_exponential_backoff_with_merge_tree(started_cluster, node, found_in_log def test_exponential_backoff_with_replicated_tree(started_cluster): prepare_cluster(True) - node_with_backoff.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM notexist_table) SETTINGS allow_nondeterministic_mutations=1" - ) + node_with_backoff.query(FAILING_MUTATION_QUERY) assert node_with_backoff.wait_for_log_line(REPLICATED_POSTPONE_MUTATION_LOG) assert not node_no_backoff.contains_in_log(REPLICATED_POSTPONE_MUTATION_LOG) @@ -114,7 +109,7 @@ def test_exponential_backoff_create_dependent_table(started_cluster): # Executing incorrect mutation. node_with_backoff.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM dep_table) SETTINGS allow_nondeterministic_mutations=1" + "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM dep_table) SETTINGS allow_nondeterministic_mutations = 1, validate_mutation_query = 0" ) # Creating dependent table for mutation. @@ -148,9 +143,7 @@ def test_exponential_backoff_setting_override(started_cluster): node.query("INSERT INTO test_mutations SELECT * FROM system.numbers LIMIT 10") # Executing incorrect mutation. - node.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM dep_table) SETTINGS allow_nondeterministic_mutations=1" - ) + node.query(FAILING_MUTATION_QUERY) assert not node.contains_in_log(POSTPONE_MUTATION_LOG) @@ -166,9 +159,7 @@ def test_backoff_clickhouse_restart(started_cluster, replicated_table): node = node_with_backoff # Executing incorrect mutation. - node.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM dep_table) SETTINGS allow_nondeterministic_mutations=1" - ) + node.query(FAILING_MUTATION_QUERY) assert node.wait_for_log_line( REPLICATED_POSTPONE_MUTATION_LOG if replicated_table else POSTPONE_MUTATION_LOG ) @@ -193,11 +184,10 @@ def test_no_backoff_after_killing_mutation(started_cluster, replicated_table): node = node_with_backoff # Executing incorrect mutation. - node.query( - "ALTER TABLE test_mutations DELETE WHERE x IN (SELECT x FROM dep_table) SETTINGS allow_nondeterministic_mutations=1" - ) + node.query(FAILING_MUTATION_QUERY) + # Executing correct mutation. - node.query("ALTER TABLE test_mutations DELETE WHERE x=1") + node.query("ALTER TABLE test_mutations DELETE WHERE x=1") assert node.wait_for_log_line( REPLICATED_POSTPONE_MUTATION_LOG if replicated_table else POSTPONE_MUTATION_LOG ) diff --git a/tests/integration/test_prometheus_protocols/test.py b/tests/integration/test_prometheus_protocols/test.py index e368c841c4e..49bc7817f02 100644 --- a/tests/integration/test_prometheus_protocols/test.py +++ b/tests/integration/test_prometheus_protocols/test.py @@ -20,7 +20,7 @@ node = cluster.add_instance( def execute_query_on_prometheus_writer(query, timestamp): return execute_query_impl( - cluster.get_instance_ip(cluster.prometheus_writer_host), + cluster.prometheus_writer_ip, cluster.prometheus_writer_port, "/api/v1/query", query, @@ -30,7 +30,7 @@ def execute_query_on_prometheus_writer(query, timestamp): def execute_query_on_prometheus_reader(query, timestamp): return execute_query_impl( - cluster.get_instance_ip(cluster.prometheus_reader_host), + cluster.prometheus_reader_ip, cluster.prometheus_reader_port, "/api/v1/query", query, diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py index ec098d7ac54..e810b613290 100644 --- a/tests/integration/test_storage_mongodb/test.py +++ b/tests/integration/test_storage_mongodb/test.py @@ -395,7 +395,7 @@ def test_secure_connection_uri(started_cluster): simple_mongo_table.insert_many(data) node = started_cluster.instances["node"] node.query( - "CREATE OR REPLACE TABLE test_secure_connection_uri(key UInt64, data String) ENGINE = MongoDB('mongodb://root:clickhouse@mongo_secure:27017/test?tls=true&tlsAllowInvalidCertificates=true&tlsAllowInvalidHostnames=true', 'test_secure_connection_uri')" + "CREATE OR REPLACE TABLE test_secure_connection_uri(key UInt64, data String) ENGINE = MongoDB('mongodb://root:clickhouse@mongo_secure:27017/test?tls=true&tlsAllowInvalidCertificates=true&tlsAllowInvalidHostnames=true&authSource=admin', 'test_secure_connection_uri')" ) assert node.query("SELECT COUNT() FROM test_secure_connection_uri") == "100\n" diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 62afc0b1c1d..aecc1df491b 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -407,6 +407,8 @@ def test_failed_retry(started_cluster, mode, engine_name): additional_settings={ "s3queue_loading_retries": retries_num, "keeper_path": keeper_path, + "polling_max_timeout_ms": 5000, + "polling_backoff_ms": 1000, }, engine_name=engine_name, ) @@ -852,6 +854,8 @@ def test_multiple_tables_streaming_sync_distributed(started_cluster, mode): additional_settings={ "keeper_path": keeper_path, "s3queue_buckets": 2, + "polling_max_timeout_ms": 2000, + "polling_backoff_ms": 1000, **({"s3queue_processing_threads_num": 1} if mode == "ordered" else {}), }, ) @@ -929,6 +933,8 @@ def test_max_set_age(started_cluster): "cleanup_interval_min_ms": max_age / 3, "cleanup_interval_max_ms": max_age / 3, "loading_retries": 0, + "polling_max_timeout_ms": 5000, + "polling_backoff_ms": 1000, "processing_threads_num": 1, "loading_retries": 0, }, @@ -1423,6 +1429,8 @@ def test_shards_distributed(started_cluster, mode, processing_threads): "keeper_path": keeper_path, "s3queue_processing_threads_num": processing_threads, "s3queue_buckets": shards_num, + "polling_max_timeout_ms": 1000, + "polling_backoff_ms": 0, }, ) i += 1 @@ -1673,6 +1681,8 @@ def test_processed_file_setting_distributed(started_cluster, processing_threads) "s3queue_processing_threads_num": processing_threads, "s3queue_last_processed_path": f"{files_path}/test_5.csv", "s3queue_buckets": 2, + "polling_max_timeout_ms": 2000, + "polling_backoff_ms": 1000, }, ) diff --git a/tests/integration/test_table_db_num_limit/config/config.xml b/tests/integration/test_table_db_num_limit/config/config.xml index 9a573b158fe..88438d51b94 100644 --- a/tests/integration/test_table_db_num_limit/config/config.xml +++ b/tests/integration/test_table_db_num_limit/config/config.xml @@ -1,4 +1,20 @@ + + + + + node1 + 9000 + + + node2 + 9000 + + + + + + 10 10 10 diff --git a/tests/integration/test_table_db_num_limit/config/config1.xml b/tests/integration/test_table_db_num_limit/config/config1.xml new file mode 100644 index 00000000000..73b695f3cd6 --- /dev/null +++ b/tests/integration/test_table_db_num_limit/config/config1.xml @@ -0,0 +1,4 @@ + + 5 + + diff --git a/tests/integration/test_table_db_num_limit/config/config2.xml b/tests/integration/test_table_db_num_limit/config/config2.xml new file mode 100644 index 00000000000..e46ca03d70f --- /dev/null +++ b/tests/integration/test_table_db_num_limit/config/config2.xml @@ -0,0 +1,4 @@ + + 3 + + diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py index b3aff6ddca2..53a644a262c 100644 --- a/tests/integration/test_table_db_num_limit/test.py +++ b/tests/integration/test_table_db_num_limit/test.py @@ -1,11 +1,22 @@ import pytest -from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node", main_configs=["config/config.xml"]) +node = cluster.add_instance( + "node1", + with_zookeeper=True, + macros={"replica": "r1"}, + main_configs=["config/config.xml", "config/config1.xml"], +) + +node2 = cluster.add_instance( + "node2", + with_zookeeper=True, + macros={"replica": "r2"}, + main_configs=["config/config.xml", "config/config2.xml"], +) @pytest.fixture(scope="module") @@ -24,10 +35,9 @@ def test_table_db_limit(started_cluster): for i in range(9): node.query("create database db{}".format(i)) - with pytest.raises(QueryRuntimeException) as exp_info: - node.query("create database db_exp".format(i)) - - assert "TOO_MANY_DATABASES" in str(exp_info) + assert "TOO_MANY_DATABASES" in node.query_and_get_error( + "create database db_exp".format(i) + ) for i in range(10): node.query("create table t{} (a Int32) Engine = Log".format(i)) @@ -35,13 +45,72 @@ def test_table_db_limit(started_cluster): # This checks that system tables are not accounted in the number of tables. node.query("system flush logs") + # Regular tables for i in range(10): node.query("drop table t{}".format(i)) for i in range(10): node.query("create table t{} (a Int32) Engine = Log".format(i)) - with pytest.raises(QueryRuntimeException) as exp_info: - node.query("create table default.tx (a Int32) Engine = Log") + assert "TOO_MANY_TABLES" in node.query_and_get_error( + "create table default.tx (a Int32) Engine = Log" + ) - assert "TOO_MANY_TABLES" in str(exp_info) + # Dictionaries + for i in range(10): + node.query( + "create dictionary d{} (a Int32) primary key a source(null()) layout(flat()) lifetime(1000)".format( + i + ) + ) + + assert "TOO_MANY_TABLES" in node.query_and_get_error( + "create dictionary dx (a Int32) primary key a source(null()) layout(flat()) lifetime(1000)" + ) + + # Replicated tables + for i in range(10): + node.query("drop table t{}".format(i)) + + for i in range(3): + node.query( + "create table t{} on cluster 'cluster' (a Int32) Engine = ReplicatedMergeTree('/clickhouse/tables/t{}', '{{replica}}') order by a".format( + i, i + ) + ) + + # Test limit on other replica + assert "Too many replicated tables" in node2.query_and_get_error( + "create table tx (a Int32) Engine = ReplicatedMergeTree('/clickhouse/tables/tx', '{replica}') order by a" + ) + + for i in range(3, 5): + node.query( + "create table t{} (a Int32) Engine = ReplicatedMergeTree('/clickhouse/tables/t{}', '{{replica}}') order by a".format( + i, i + ) + ) + + assert "Too many replicated tables" in node.query_and_get_error( + "create table tx (a Int32) Engine = ReplicatedMergeTree('/clickhouse/tables/tx', '{replica}') order by a" + ) + + # Checks that replicated tables are also counted as regular tables + for i in range(5, 10): + node.query("create table t{} (a Int32) Engine = Log".format(i)) + + assert "TOO_MANY_TABLES" in node.query_and_get_error( + "create table tx (a Int32) Engine = Log" + ) + + # Cleanup + for i in range(10): + node.query("drop table t{} sync".format(i)) + for i in range(3): + node2.query("drop table t{} sync".format(i)) + node.query("system drop replica 'r1' from ZKPATH '/clickhouse/tables/tx'") + node.query("system drop replica 'r2' from ZKPATH '/clickhouse/tables/tx'") + for i in range(9): + node.query("drop database db{}".format(i)) + for i in range(10): + node.query("drop dictionary d{}".format(i)) diff --git a/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql b/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql index 1d033b7aa47..c840bba6871 100644 --- a/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql +++ b/tests/queries/0_stateless/00755_avg_value_size_hint_passing.sql @@ -5,7 +5,7 @@ SET max_block_size = 1000; SET max_memory_usage = 1000000000; INSERT INTO size_hint SELECT arrayMap(x -> 'Hello', range(1000)) FROM numbers(10000); -SET max_memory_usage = 100000000, max_threads = 2; +SET max_memory_usage = 105000000, max_threads = 2; SELECT count(), sum(length(s)) FROM size_hint; DROP TABLE size_hint; diff --git a/tests/queries/0_stateless/00996_limit_with_ties.reference b/tests/queries/0_stateless/00996_limit_with_ties.reference index ecbbaa76f7a..43f3a21fcda 100644 --- a/tests/queries/0_stateless/00996_limit_with_ties.reference +++ b/tests/queries/0_stateless/00996_limit_with_ties.reference @@ -53,3 +53,5 @@ 100 100 100 +12 +12 diff --git a/tests/queries/0_stateless/00996_limit_with_ties.sql b/tests/queries/0_stateless/00996_limit_with_ties.sql index 8d7aedc2cb0..4ba2ccfc57d 100644 --- a/tests/queries/0_stateless/00996_limit_with_ties.sql +++ b/tests/queries/0_stateless/00996_limit_with_ties.sql @@ -35,4 +35,24 @@ select count() from (select number, number < 100 from numbers(2000) order by num SET max_block_size = 5; select count() from (select number < 100, number from numbers(2000) order by number < 100 desc limit 10 with ties); +SELECT count() FROM (WITH data AS ( + SELECT * FROM numbers(0, 10) + UNION ALL + SELECT * FROM numbers(10, 10) +) +SELECT number div 10 AS ten, number +FROM data +ORDER BY ten +LIMIT 8,6 WITH TIES); + +SELECT count() FROM (WITH data AS ( + SELECT * FROM numbers(0, 10) + UNION ALL + SELECT * FROM numbers(10, 10) +) +SELECT number div 11 AS eleven, number +FROM data +ORDER BY eleven +LIMIT 8,6 WITH TIES); + DROP TABLE ties; diff --git a/tests/queries/0_stateless/01655_plan_optimizations.reference b/tests/queries/0_stateless/01655_plan_optimizations.reference index edf93b4b39f..33c2593ccfc 100644 --- a/tests/queries/0_stateless/01655_plan_optimizations.reference +++ b/tests/queries/0_stateless/01655_plan_optimizations.reference @@ -82,12 +82,12 @@ Filter column: notEquals(__table1.y, 0_UInt8) 9 10 > one condition of filter should be pushed down after aggregating, other two conditions are ANDed Filter column -FUNCTION and(minus(s, 8) :: 5, minus(s, 4) :: 2) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4)) +FUNCTION and(minus(s, 8) :: 3, minus(s, 4) :: 5) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4)) Aggregating Filter column: notEquals(y, 0) > (analyzer) one condition of filter should be pushed down after aggregating, other two conditions are ANDed Filter column -FUNCTION and(minus(__table1.s, 8_UInt8) :: 1, minus(__table1.s, 4_UInt8) :: 2) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8)) +FUNCTION and(minus(__table1.s, 8_UInt8) :: 3, minus(__table1.s, 4_UInt8) :: 5) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8)) Aggregating Filter column: notEquals(__table1.y, 0_UInt8) 0 1 diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh index 42cdac8c01f..04ab9bbd11c 100755 --- a/tests/queries/0_stateless/01655_plan_optimizations.sh +++ b/tests/queries/0_stateless/01655_plan_optimizations.sh @@ -89,14 +89,14 @@ $CLICKHOUSE_CLIENT --enable_analyzer=0 --convert_query_to_cnf=0 -q " select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y ) where y != 0 and s - 8 and s - 4 settings enable_optimize_predicate_expression=0" | - grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 8) :: 5, minus(s, 4) :: 2) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4))" + grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 8) :: 3, minus(s, 4) :: 5) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4))" echo "> (analyzer) one condition of filter should be pushed down after aggregating, other two conditions are ANDed" $CLICKHOUSE_CLIENT --enable_analyzer=1 --convert_query_to_cnf=0 -q " explain actions = 1 select s, y from ( select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y ) where y != 0 and s - 8 and s - 4 settings enable_optimize_predicate_expression=0" | - grep -o "Aggregating\|Filter column\|Filter column: notEquals(__table1.y, 0_UInt8)\|FUNCTION and(minus(__table1.s, 8_UInt8) :: 1, minus(__table1.s, 4_UInt8) :: 2) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8))" + grep -o "Aggregating\|Filter column\|Filter column: notEquals(__table1.y, 0_UInt8)\|FUNCTION and(minus(__table1.s, 8_UInt8) :: 3, minus(__table1.s, 4_UInt8) :: 5) -> and(notEquals(__table1.y, 0_UInt8), minus(__table1.s, 8_UInt8), minus(__table1.s, 4_UInt8))" $CLICKHOUSE_CLIENT -q " select s, y from ( select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y diff --git a/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference b/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference index 4da8a9784dd..fb92bdda821 100644 --- a/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference +++ b/tests/queries/0_stateless/02812_from_to_utc_timestamp.reference @@ -3,3 +3,10 @@ 3 2023-03-16 12:22:33 2023-03-16 10:22:33.000 2023-03-16 03:22:33 2023-03-16 19:22:33.123 2024-02-24 10:22:33 2024-02-24 12:22:33 2024-10-24 09:22:33 2024-10-24 13:22:33 +2024-10-24 16:22:33 2024-10-24 06:22:33 +leap year: 2024-02-29 16:22:33 2024-02-29 06:22:33 +non-leap year: 2023-03-01 16:22:33 2023-03-01 06:22:33 +leap year: 2024-02-29 04:22:33 2024-02-29 19:22:33 +non-leap year: 2023-03-01 04:22:33 2023-02-28 19:22:33 +timezone with half-hour offset: 2024-02-29 00:52:33 2024-02-29 21:52:33 +jump over a year: 2024-01-01 04:01:01 2023-12-31 20:01:01 diff --git a/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh b/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh index 835dab8af57..9eb4484ace0 100755 --- a/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh +++ b/tests/queries/0_stateless/02812_from_to_utc_timestamp.sh @@ -15,4 +15,13 @@ $CLICKHOUSE_CLIENT -q "select x, to_utc_timestamp(toDateTime('2023-03-16 11:22:3 # timestamp convert between DST timezone and UTC $CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-02-24 11:22:33'), 'Europe/Madrid')" $CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'Europe/Madrid')" -$CLICKHOUSE_CLIENT -q "drop table test_tbl" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "select to_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-10-24 11:22:33'), 'EST')" + +$CLICKHOUSE_CLIENT -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-02-29 11:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'leap year:', to_utc_timestamp(toDateTime('2024-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2024-03-01 00:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'non-leap year:', to_utc_timestamp(toDateTime('2023-02-28 23:22:33'), 'EST'), from_utc_timestamp(toDateTime('2023-03-01 00:22:33'), 'EST')" +$CLICKHOUSE_CLIENT -q "select 'timezone with half-hour offset:', to_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide'), from_utc_timestamp(toDateTime('2024-02-29 11:22:33'), 'Australia/Adelaide')" +$CLICKHOUSE_CLIENT -q "select 'jump over a year:', to_utc_timestamp(toDateTime('2023-12-31 23:01:01'), 'EST'), from_utc_timestamp(toDateTime('2024-01-01 01:01:01'), 'EST')" + +$CLICKHOUSE_CLIENT -q "drop table test_tbl" diff --git a/tests/queries/0_stateless/02993_lazy_index_loading.reference b/tests/queries/0_stateless/02993_lazy_index_loading.reference index 08f07a92815..445cb22b3ef 100644 --- a/tests/queries/0_stateless/02993_lazy_index_loading.reference +++ b/tests/queries/0_stateless/02993_lazy_index_loading.reference @@ -1,4 +1,18 @@ -100000000 100000000 -0 0 +Row 1: +────── +round(primary_key_bytes_in_memory, -7): 100000000 -- 100.00 million +round(primary_key_bytes_in_memory_allocated, -7): 100000000 -- 100.00 million +Row 1: +────── +primary_key_bytes_in_memory: 0 +primary_key_bytes_in_memory_allocated: 0 1 -100000000 100000000 +Row 1: +────── +primary_key_bytes_in_memory: 0 +primary_key_bytes_in_memory_allocated: 0 +1 +Row 1: +────── +round(primary_key_bytes_in_memory, -7): 100000000 -- 100.00 million +round(primary_key_bytes_in_memory_allocated, -7): 100000000 -- 100.00 million diff --git a/tests/queries/0_stateless/02993_lazy_index_loading.sql b/tests/queries/0_stateless/02993_lazy_index_loading.sql index ffb4b7547bf..5d4233e7d5e 100644 --- a/tests/queries/0_stateless/02993_lazy_index_loading.sql +++ b/tests/queries/0_stateless/02993_lazy_index_loading.sql @@ -3,17 +3,26 @@ CREATE TABLE test (s String) ENGINE = MergeTree ORDER BY s SETTINGS index_granul SET optimize_trivial_insert_select = 1; INSERT INTO test SELECT randomString(1000) FROM numbers(100000); -SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table = 'test'; +SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table = 'test' FORMAT Vertical; DETACH TABLE test; SET max_memory_usage = '50M'; ATTACH TABLE test; -SELECT primary_key_bytes_in_memory, primary_key_bytes_in_memory_allocated FROM system.parts WHERE database = currentDatabase() AND table = 'test'; +SELECT primary_key_bytes_in_memory, primary_key_bytes_in_memory_allocated FROM system.parts WHERE database = currentDatabase() AND table = 'test' FORMAT Vertical; SET max_memory_usage = '200M'; + +-- Run a query that doesn use indexes SELECT s != '' FROM test LIMIT 1; -SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table = 'test'; +-- Check that index was not loaded +SELECT primary_key_bytes_in_memory, primary_key_bytes_in_memory_allocated FROM system.parts WHERE database = currentDatabase() AND table = 'test' FORMAT Vertical; + +-- Run a query that uses PK index +SELECT s != '' FROM test WHERE s < '9999999999' LIMIT 1; + +-- Check that index was loaded +SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table = 'test' FORMAT Vertical; DROP TABLE test; diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference index 79ebc7a5c0c..db8d26ccfea 100644 --- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference +++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference @@ -83,7 +83,7 @@ QUERY id: 0 FUNCTION id: 4, function_name: tuple, function_type: ordinary, result_type: Tuple(Nullable(UInt64)) ARGUMENTS LIST id: 5, nodes: 1 - FUNCTION id: 6, function_name: sum, function_type: aggregate, nulls_action : IGNORE_NULLS, result_type: Nullable(UInt64) + FUNCTION id: 6, function_name: sum, function_type: aggregate, result_type: Nullable(UInt64) ARGUMENTS LIST id: 7, nodes: 1 FUNCTION id: 8, function_name: if, function_type: ordinary, result_type: Nullable(UInt8) diff --git a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference index 80f4e309505..d0a3e7b02ae 100644 --- a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference +++ b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.reference @@ -163,17 +163,21 @@ Positions: 4 2 0 1 Filter (( + (JOIN actions + Change column names to column identifiers))) Header: __table1.id UInt64 __table1.value String - Filter column: and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) (removed) + AND column: equals(__table1.id, 5_UInt8) Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value String : 1 + COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 1 + FUNCTION equals(id : 0, 5_UInt8 :: 1) -> equals(__table1.id, 5_UInt8) UInt8 : 2 + Positions: 2 0 2 + Filter column: and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) (removed) + Actions: INPUT : 2 -> value String : 0 + INPUT : 1 -> id UInt64 : 1 COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 2 - COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 3 - ALIAS id : 0 -> __table1.id UInt64 : 4 - ALIAS value :: 1 -> __table1.value String : 5 - FUNCTION equals(id : 0, 6_UInt8 :: 2) -> equals(__table1.id, 6_UInt8) UInt8 : 1 - FUNCTION equals(id :: 0, 5_UInt8 :: 3) -> equals(__table1.id, 5_UInt8) UInt8 : 2 - FUNCTION and(equals(__table1.id, 5_UInt8) :: 2, equals(__table1.id, 6_UInt8) :: 1) -> and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) UInt8 : 3 - Positions: 3 4 5 + INPUT : 0 -> equals(__table1.id, 5_UInt8) UInt8 : 3 + ALIAS value :: 0 -> __table1.value String : 4 + ALIAS id : 1 -> __table1.id UInt64 : 0 + FUNCTION equals(id :: 1, 6_UInt8 :: 2) -> equals(__table1.id, 6_UInt8) UInt8 : 5 + FUNCTION and(equals(__table1.id, 5_UInt8) :: 3, equals(__table1.id, 6_UInt8) :: 5) -> and(equals(__table1.id, 5_UInt8), equals(__table1.id, 6_UInt8)) UInt8 : 2 + Positions: 2 0 4 ReadFromMergeTree (default.test_table_1) Header: id UInt64 value String @@ -183,17 +187,21 @@ Positions: 4 2 0 1 Filter (( + (JOIN actions + Change column names to column identifiers))) Header: __table2.id UInt64 __table2.value String - Filter column: and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) (removed) + AND column: equals(__table2.id, 6_UInt8) Actions: INPUT : 0 -> id UInt64 : 0 - INPUT : 1 -> value String : 1 + COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 1 + FUNCTION equals(id : 0, 6_UInt8 :: 1) -> equals(__table2.id, 6_UInt8) UInt8 : 2 + Positions: 2 0 2 + Filter column: and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) (removed) + Actions: INPUT : 2 -> value String : 0 + INPUT : 1 -> id UInt64 : 1 COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 2 - COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 3 - ALIAS id : 0 -> __table2.id UInt64 : 4 - ALIAS value :: 1 -> __table2.value String : 5 - FUNCTION equals(id : 0, 5_UInt8 :: 2) -> equals(__table2.id, 5_UInt8) UInt8 : 1 - FUNCTION equals(id :: 0, 6_UInt8 :: 3) -> equals(__table2.id, 6_UInt8) UInt8 : 2 - FUNCTION and(equals(__table2.id, 6_UInt8) :: 2, equals(__table2.id, 5_UInt8) :: 1) -> and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) UInt8 : 3 - Positions: 3 4 5 + INPUT : 0 -> equals(__table2.id, 6_UInt8) UInt8 : 3 + ALIAS value :: 0 -> __table2.value String : 4 + ALIAS id : 1 -> __table2.id UInt64 : 0 + FUNCTION equals(id :: 1, 5_UInt8 :: 2) -> equals(__table2.id, 5_UInt8) UInt8 : 5 + FUNCTION and(equals(__table2.id, 6_UInt8) :: 3, equals(__table2.id, 5_UInt8) :: 5) -> and(equals(__table2.id, 6_UInt8), equals(__table2.id, 5_UInt8)) UInt8 : 2 + Positions: 2 0 4 ReadFromMergeTree (default.test_table_2) Header: id UInt64 value String @@ -656,17 +664,21 @@ Positions: 4 2 0 1 __table1.value String __table2.value String __table2.id UInt64 - Filter column: and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) (removed) + AND column: equals(__table1.id, 5_UInt8) Actions: INPUT : 0 -> __table1.id UInt64 : 0 - INPUT :: 1 -> __table1.value String : 1 - INPUT :: 2 -> __table2.value String : 2 - INPUT : 3 -> __table2.id UInt64 : 3 - COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 4 - COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 5 - FUNCTION equals(__table1.id : 0, 5_UInt8 :: 4) -> equals(__table1.id, 5_UInt8) UInt8 : 6 - FUNCTION equals(__table2.id : 3, 6_UInt8 :: 5) -> equals(__table2.id, 6_UInt8) UInt8 : 4 - FUNCTION and(equals(__table1.id, 5_UInt8) :: 6, equals(__table2.id, 6_UInt8) :: 4) -> and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) UInt8 : 5 - Positions: 5 0 1 2 3 + COLUMN Const(UInt8) -> 5_UInt8 UInt8 : 1 + FUNCTION equals(__table1.id : 0, 5_UInt8 :: 1) -> equals(__table1.id, 5_UInt8) UInt8 : 2 + Positions: 2 0 2 + Filter column: and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) (removed) + Actions: INPUT :: 1 -> __table1.id UInt64 : 0 + INPUT :: 2 -> __table1.value String : 1 + INPUT :: 3 -> __table2.value String : 2 + INPUT : 4 -> __table2.id UInt64 : 3 + COLUMN Const(UInt8) -> 6_UInt8 UInt8 : 4 + INPUT : 0 -> equals(__table1.id, 5_UInt8) UInt8 : 5 + FUNCTION equals(__table2.id : 3, 6_UInt8 :: 4) -> equals(__table2.id, 6_UInt8) UInt8 : 6 + FUNCTION and(equals(__table1.id, 5_UInt8) :: 5, equals(__table2.id, 6_UInt8) :: 6) -> and(equals(__table1.id, 5_UInt8), equals(__table2.id, 6_UInt8)) UInt8 : 4 + Positions: 4 0 1 2 3 Join (JOIN FillRightFirst) Header: __table1.id UInt64 __table1.value String diff --git a/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference b/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference index 2d33f7f6683..194d3d099b3 100644 --- a/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference +++ b/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference @@ -5,9 +5,19 @@ 100000000 100000000 0 0 0 0 +Query that does not use index for table `test` +1 +0 0 +0 0 +Query that uses index in for table `test` 1 100000000 100000000 0 0 +Query that does not use index for table `test2` +1 +100000000 100000000 +0 0 +Query that uses index for table `test2` 1 100000000 100000000 100000000 100000000 diff --git a/tests/queries/0_stateless/03127_system_unload_primary_key_table.sql b/tests/queries/0_stateless/03127_system_unload_primary_key_table.sql index de99464a9e6..414e661c5ba 100644 --- a/tests/queries/0_stateless/03127_system_unload_primary_key_table.sql +++ b/tests/queries/0_stateless/03127_system_unload_primary_key_table.sql @@ -16,8 +16,18 @@ SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory SYSTEM UNLOAD PRIMARY KEY {CLICKHOUSE_DATABASE:Identifier}.test2; SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table IN ('test', 'test2'); +SELECT 'Query that does not use index for table `test`'; SELECT s != '' FROM test LIMIT 1; SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table IN ('test', 'test2'); +SELECT 'Query that uses index in for table `test`'; +SELECT s != '' FROM test WHERE s < '99999999' LIMIT 1; +SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table IN ('test', 'test2'); + +SELECT 'Query that does not use index for table `test2`'; SELECT s != '' FROM test2 LIMIT 1; SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table IN ('test', 'test2'); + +SELECT 'Query that uses index for table `test2`'; +SELECT s != '' FROM test2 WHERE s < '99999999' LIMIT 1; +SELECT round(primary_key_bytes_in_memory, -7), round(primary_key_bytes_in_memory_allocated, -7) FROM system.parts WHERE database = currentDatabase() AND table IN ('test', 'test2'); diff --git a/tests/queries/0_stateless/03173_forbid_qualify.sql b/tests/queries/0_stateless/03173_forbid_qualify.sql index 0a41385c52f..f7b05a1eb7e 100644 --- a/tests/queries/0_stateless/03173_forbid_qualify.sql +++ b/tests/queries/0_stateless/03173_forbid_qualify.sql @@ -7,5 +7,5 @@ select count() from test_qualify; -- 100 select * from test_qualify qualify row_number() over (order by number) = 50 SETTINGS enable_analyzer = 1; -- 49 select * from test_qualify qualify row_number() over (order by number) = 50 SETTINGS enable_analyzer = 0; -- { serverError NOT_IMPLEMENTED } -delete from test_qualify where number in (select number from test_qualify qualify row_number() over (order by number) = 50); -- { serverError UNFINISHED } +delete from test_qualify where number in (select number from test_qualify qualify row_number() over (order by number) = 50) SETTINGS validate_mutation_query = 0; -- { serverError UNFINISHED } select count() from test_qualify; -- 100 diff --git a/tests/queries/0_stateless/03199_merge_filters_bug.sql b/tests/queries/0_stateless/03199_merge_filters_bug.sql index ed2ec2ea217..696856c9121 100644 --- a/tests/queries/0_stateless/03199_merge_filters_bug.sql +++ b/tests/queries/0_stateless/03199_merge_filters_bug.sql @@ -1,3 +1,5 @@ +set allow_reorder_prewhere_conditions=0; + drop table if exists t1; drop table if exists t2; @@ -49,7 +51,23 @@ tmp1 AS fs1 FROM t2 LEFT JOIN tmp1 USING (fs1) - WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 0; + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 0, query_plan_merge_filters=0; + +WITH +tmp1 AS +( + SELECT + CAST(s1, 'FixedString(10)') AS fs1, + s2 AS sector, + s3 + FROM t1 + WHERE (s3 != 'test') +) + SELECT + fs1 + FROM t2 + LEFT JOIN tmp1 USING (fs1) + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 1, query_plan_merge_filters=1; optimize table t1 final; @@ -67,4 +85,20 @@ tmp1 AS fs1 FROM t2 LEFT JOIN tmp1 USING (fs1) - WHERE (fs1 IN ('test')); + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 0, query_plan_merge_filters=0; + +WITH +tmp1 AS +( + SELECT + CAST(s1, 'FixedString(10)') AS fs1, + s2 AS sector, + s3 + FROM t1 + WHERE (s3 != 'test') +) + SELECT + fs1 + FROM t2 + LEFT JOIN tmp1 USING (fs1) + WHERE (fs1 IN ('test')) SETTINGS enable_multiple_prewhere_read_steps = 1, query_plan_merge_filters=1; diff --git a/tests/queries/0_stateless/03203_system_query_metric_log.reference b/tests/queries/0_stateless/03203_system_query_metric_log.reference index 940b0c4e178..fa8e27a7e90 100644 --- a/tests/queries/0_stateless/03203_system_query_metric_log.reference +++ b/tests/queries/0_stateless/03203_system_query_metric_log.reference @@ -23,8 +23,8 @@ --Interval 123: check that the SleepFunctionCalls, SleepFunctionMilliseconds and ProfileEvent_SleepFunctionElapsedMicroseconds are correct 1 --Check that a query_metric_log_interval=0 disables the collection -0 +1 -Check that a query which execution time is less than query_metric_log_interval is never collected -0 +1 --Check that there is a final event when queries finish -3 +1 diff --git a/tests/queries/0_stateless/03203_system_query_metric_log.sh b/tests/queries/0_stateless/03203_system_query_metric_log.sh index bf94be79d7c..abcd14c8e5d 100755 --- a/tests/queries/0_stateless/03203_system_query_metric_log.sh +++ b/tests/queries/0_stateless/03203_system_query_metric_log.sh @@ -84,17 +84,17 @@ check_log 123 # query_metric_log_interval=0 disables the collection altogether $CLICKHOUSE_CLIENT -m -q """ SELECT '--Check that a query_metric_log_interval=0 disables the collection'; - SELECT count() FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_0' + SELECT count() == 0 FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_0' """ # a quick query that takes less than query_metric_log_interval is never collected $CLICKHOUSE_CLIENT -m -q """ SELECT '-Check that a query which execution time is less than query_metric_log_interval is never collected'; - SELECT count() FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_fast' + SELECT count() == 0 FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_fast' """ # a query that takes more than query_metric_log_interval is collected including the final row $CLICKHOUSE_CLIENT -m -q """ SELECT '--Check that there is a final event when queries finish'; - SELECT count() FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_1000' + SELECT count() > 2 FROM system.query_metric_log WHERE event_date >= yesterday() AND query_id = '${query_prefix}_1000' """ diff --git a/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.reference b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.reference new file mode 100644 index 00000000000..1f242fa6f00 --- /dev/null +++ b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.reference @@ -0,0 +1,3 @@ +3 +3 +3 diff --git a/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.sql b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.sql new file mode 100644 index 00000000000..72f9781ed45 --- /dev/null +++ b/tests/queries/0_stateless/03222_ignore_nulls_query_tree_elimination.sql @@ -0,0 +1,51 @@ +#!/usr/bin/env -S ${HOME}/clickhouse-client --queries-file + +DROP TABLE IF EXISTS with_fill_date__fuzz_0; + +CREATE TABLE with_fill_date__fuzz_0 +( + `d` Date, + `d32` Nullable(Int32), + `d33` Int32 +) +ENGINE = Memory; + + +INSERT INTO with_fill_date__fuzz_0 VALUES (toDate('2020-03-03'), 1, 3), (toDate('2020-03-03'), NULL, 3), (toDate('2020-02-05'), 1, 1); + + +SELECT count() +FROM with_fill_date__fuzz_0 +ORDER BY + count(), + count() IGNORE NULLS, + max(d) +WITH FILL STEP toIntervalDay(10) +; + + +SELECT count() +FROM with_fill_date__fuzz_0 +ORDER BY + any(d32) RESPECT NULLS, + any_respect_nulls(d32), + max(d) +WITH FILL STEP toIntervalDay(10) +; + + +SELECT count() +FROM with_fill_date__fuzz_0 +ORDER BY + any(d32), + any(d32) IGNORE NULLS, + any(d32) RESPECT NULLS, + any_respect_nulls(d32) IGNORE NULLS, + any_respect_nulls(d32), + sum(d33), + sum(d33) IGNORE NULLS, + max(d) +WITH FILL STEP toIntervalDay(10) +; + + diff --git a/tests/queries/0_stateless/03224_arrayUnion.reference b/tests/queries/0_stateless/03224_arrayUnion.reference index b900b6cdb0a..9b871234d27 100644 --- a/tests/queries/0_stateless/03224_arrayUnion.reference +++ b/tests/queries/0_stateless/03224_arrayUnion.reference @@ -41,3 +41,13 @@ [1,2,3,4,5,10,20] ------- [1,2,3] +------- +[10,-2,1] ['hello','hi'] [3,2,1,NULL] +------- +------- +[1] +------- +[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256] +199999 +------- +[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] diff --git a/tests/queries/0_stateless/03224_arrayUnion.sql b/tests/queries/0_stateless/03224_arrayUnion.sql index dedbacad906..14a9192f39a 100644 --- a/tests/queries/0_stateless/03224_arrayUnion.sql +++ b/tests/queries/0_stateless/03224_arrayUnion.sql @@ -35,4 +35,23 @@ SELECT arraySort(arrayUnion([NULL, NULL, NULL, 1], [1, NULL, NULL], [1, 2, 3, NU select '-------'; SELECT arraySort(arrayUnion([1, 1, 1, 2, 3], [2, 2, 4], [5, 10, 20])); select '-------'; -SELECT arraySort(arrayUnion([1, 2], [1, 3], [])), +SELECT arraySort(arrayUnion([1, 2], [1, 3], [])); +select '-------'; +-- example from docs +SELECT + arrayUnion([-2, 1], [10, 1], [-2], []) as num_example, + arrayUnion(['hi'], [], ['hello', 'hi']) as str_example, + arrayUnion([1, 3, NULL], [2, 3, NULL]) as null_example; +select '-------'; +--mix of types +SELECT arrayUnion([1], [-2], [1.1, 'hi'], [NULL, 'hello', []]); -- {serverError NO_COMMON_TYPE} +select '-------'; +SELECT arrayUnion([1]); +SELECT arrayUnion(); -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH} +select '-------'; +--bigger arrays +SELECT arraySort(arrayUnion(range(1, 256), range(2, 257))); +SELECT length(arrayUnion(range(1, 100000), range(9999, 200000))); +select '-------'; +--bigger number of arguments +SELECT arraySort(arrayUnion([1, 2], [1, 3], [1, 4], [1, 5], [1, 6], [1, 7], [1, 8], [1, 9], [1, 10], [1, 11], [1, 12], [1, 13], [1, 14], [1, 15], [1, 16], [1, 17], [1, 18], [1, 19])); diff --git a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference index 23cbe2bfdec..ccb315b8305 100644 --- a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference +++ b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.reference @@ -1,6 +1,6 @@ quantileExactWeightedInterpolated -0 0 0 Decimal(38, 8) --25.5 -8.49999999 -5.1 Decimal(38, 8) +0 0 0 25 2024-02-20 Decimal(38, 8) +-25.5 -8.49999999 -5.1 12.25 2024-01-25 Decimal(38, 8) 0 0 0 10 3.33333333 2 20 6.66666666 4 @@ -10,11 +10,14 @@ quantileExactWeightedInterpolated [-50,-40,-30,-20,-10,0,10,20,30,40,50] [-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] [-10,-8,-6,-4,-2,0,2,4,6,8,10] +[0,5,10,15,20,25,30,35,40,45,50] +['2024-01-01','2024-01-11','2024-01-21','2024-01-31','2024-02-10','2024-02-20','2024-03-01','2024-03-11','2024-03-21','2024-03-31','2024-04-10'] quantileExactWeightedInterpolatedState [10000.6,20000.2,29999.8,39999.4] Test with filter that returns no rows -0 0 0 +0 0 0 nan 1970-01-01 +0 0 0 nan 1970-01-01 Test with dynamic weights -21 7 4.2 +21 7 4.2 35.5 2024-03-12 Test with all weights set to 0 -0 0 0 +0 0 0 nan 1970-01-01 diff --git a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql index dba16eae22a..a64b46e751b 100644 --- a/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql +++ b/tests/queries/0_stateless/03240_quantile_exact_weighted_interpolated.sql @@ -5,16 +5,28 @@ CREATE TABLE decimal a Decimal32(4), b Decimal64(8), c Decimal128(8), + f Float64, + d Date, w UInt64 ) ENGINE = Memory; -INSERT INTO decimal (a, b, c, w) -SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5, number +INSERT INTO decimal (a, b, c, f, d, w) +SELECT toDecimal32(number - 50, 4), toDecimal64(number - 50, 8) / 3, toDecimal128(number - 50, 8) / 5, number/2, addDays(toDate('2024-01-01'), number), number FROM system.numbers LIMIT 101; SELECT 'quantileExactWeightedInterpolated'; -SELECT medianExactWeightedInterpolated(a, 1), medianExactWeightedInterpolated(b, 2), medianExactWeightedInterpolated(c, 3) as x, toTypeName(x) FROM decimal; -SELECT quantileExactWeightedInterpolated(a, 1), quantileExactWeightedInterpolated(b, 2), quantileExactWeightedInterpolated(c, 3) as x, toTypeName(x) FROM decimal WHERE a < 0; +SELECT medianExactWeightedInterpolated(a, 1), + medianExactWeightedInterpolated(b, 2), + medianExactWeightedInterpolated(c, 3) as x, + medianExactWeightedInterpolated(f, 4), + medianExactWeightedInterpolated(d, 5), + toTypeName(x) FROM decimal; +SELECT quantileExactWeightedInterpolated(a, 1), + quantileExactWeightedInterpolated(b, 2), + quantileExactWeightedInterpolated(c, 3) as x, + quantileExactWeightedInterpolated(f, 4), + quantileExactWeightedInterpolated(d, 5), + toTypeName(x) FROM decimal WHERE a < 0; SELECT quantileExactWeightedInterpolated(0.0)(a, 1), quantileExactWeightedInterpolated(0.0)(b, 2), quantileExactWeightedInterpolated(0.0)(c, 3) FROM decimal WHERE a >= 0; SELECT quantileExactWeightedInterpolated(0.2)(a, 1), quantileExactWeightedInterpolated(0.2)(b, 2), quantileExactWeightedInterpolated(0.2)(c, 3) FROM decimal WHERE a >= 0; SELECT quantileExactWeightedInterpolated(0.4)(a, 1), quantileExactWeightedInterpolated(0.4)(b, 2), quantileExactWeightedInterpolated(0.4)(c, 3) FROM decimal WHERE a >= 0; @@ -24,6 +36,8 @@ SELECT quantileExactWeightedInterpolated(1.0)(a, 1), quantileExactWeightedInterp SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(a, 1) FROM decimal; SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(b, 2) FROM decimal; SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(c, 3) FROM decimal; +SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(f, 4) FROM decimal; +SELECT quantilesExactWeightedInterpolated(0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)(d, 5) FROM decimal; SELECT 'quantileExactWeightedInterpolatedState'; SELECT quantilesExactWeightedInterpolatedMerge(0.2, 0.4, 0.6, 0.8)(x) @@ -34,12 +48,29 @@ FROM ); SELECT 'Test with filter that returns no rows'; -SELECT medianExactWeightedInterpolated(a, 1), medianExactWeightedInterpolated(b, 2), medianExactWeightedInterpolated(c, 3) FROM decimal WHERE a > 1000; +SELECT medianExactWeightedInterpolated(a, 1), + medianExactWeightedInterpolated(b, 2), + medianExactWeightedInterpolated(c, 3), + medianExactWeightedInterpolated(f, 4), + medianExactWeightedInterpolated(d, 5) FROM decimal WHERE a > 1000; +SELECT quantileExactWeightedInterpolated(a, 1), + quantileExactWeightedInterpolated(b, 2), + quantileExactWeightedInterpolated(c, 3), + quantileExactWeightedInterpolated(f, 4), + quantileExactWeightedInterpolated(d, 5) FROM decimal WHERE d < toDate('2024-01-01'); SELECT 'Test with dynamic weights'; -SELECT medianExactWeightedInterpolated(a, w), medianExactWeightedInterpolated(b, w), medianExactWeightedInterpolated(c, w) FROM decimal; +SELECT medianExactWeightedInterpolated(a, w), + medianExactWeightedInterpolated(b, w), + medianExactWeightedInterpolated(c, w), + medianExactWeightedInterpolated(f, w), + medianExactWeightedInterpolated(d, w) FROM decimal; SELECT 'Test with all weights set to 0'; -SELECT medianExactWeightedInterpolated(a, 0), medianExactWeightedInterpolated(b, 0), medianExactWeightedInterpolated(c, 0) FROM decimal; +SELECT medianExactWeightedInterpolated(a, 0), + medianExactWeightedInterpolated(b, 0), + medianExactWeightedInterpolated(c, 0), + medianExactWeightedInterpolated(f, 0), + medianExactWeightedInterpolated(d, 0) FROM decimal; DROP TABLE IF EXISTS decimal; diff --git a/tests/queries/0_stateless/03248_max_parts_to_move.reference b/tests/queries/0_stateless/03248_max_parts_to_move.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03248_max_parts_to_move.sql b/tests/queries/0_stateless/03248_max_parts_to_move.sql new file mode 100644 index 00000000000..aed41021e94 --- /dev/null +++ b/tests/queries/0_stateless/03248_max_parts_to_move.sql @@ -0,0 +1,23 @@ +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t (x Int32) ENGINE = MergeTree ORDER BY x; +CREATE TABLE t2 (x Int32) ENGINE = MergeTree ORDER BY x; + +SYSTEM STOP MERGES t; + +SET max_insert_block_size = 1; +SET min_insert_block_size_rows = 1; +SET max_block_size = 1; + +SET max_parts_to_move = 5; +INSERT INTO t SELECT number from numbers(10); + +ALTER TABLE t MOVE PARTITION tuple() TO TABLE t2; -- { serverError TOO_MANY_PARTS } + +SET max_parts_to_move = 15; + +ALTER TABLE t MOVE PARTITION tuple() TO TABLE t2; + +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference index 063b76b152c..e4be64155d1 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.reference @@ -1,14 +1,44 @@ +parseDateTime64InJodaSyntax +2077-10-09 10:30:10.123 +1970-01-01 08:00:00 +1970-01-01 08:00:01 +2024-01-02 00:00:00 +1970-01-01 10:30:50 +2025-01-01 15:30:10.123456 +2024-01-01 00:00:01.123456 2024-10-09 10:30:10.123 2024-10-09 10:30:10.123456 2024-10-10 02:30:10.123456 +2024-11-05 17:02:03.123456 2024-10-10 01:30:10.123456 +2024-10-09 08:00:10.123456 +2024-09-10 10:30:10.123 +2024-09-10 10:30:10.000999999 +2024-10-10 03:15:10.123456 +2024-03-01 03:23:34 +2024-02-29 15:22:33 +2023-03-01 15:22:33 +2024-03-04 16:22:33 +2023-03-04 16:22:33 +parseDateTime64InJodaSyntaxOrZero 2024-10-09 10:30:10.123 2024-10-09 10:30:10.123456 1970-01-01 08:00:00.000000000 2024-10-10 02:30:10.123456 2024-10-10 01:30:10.123456 +2024-09-10 10:30:10.123 +1970-01-01 08:00:00.000 +1970-01-01 08:00:00 +1970-01-01 08:00:00 +1970-01-01 08:00:00.000 +parseDateTime64InJodaSyntaxOrNull 2024-10-09 10:30:10.123 2024-10-09 10:30:10.123456 \N 2024-10-10 02:30:10.123456 2024-10-10 01:30:10.123456 +2024-09-10 10:30:10.123 +\N +\N +\N +1970-01-01 00:00:00 diff --git a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql index 9ea854bc324..7e024288f1c 100644 --- a/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql +++ b/tests/queries/0_stateless/03252_parse_datetime64_in_joda_syntax.sql @@ -1,19 +1,54 @@ set session_timezone = 'Asia/Shanghai'; +select 'parseDateTime64InJodaSyntax'; +select parseDateTime64InJodaSyntax('', ''); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +select parseDateTime64InJodaSyntax('2077-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntax('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('+0000', 'Z'); +select parseDateTime64InJodaSyntax('08:01', 'HH:ss'); +select parseDateTime64InJodaSyntax('2024-01-02', 'yyyy-MM-dd'); +select parseDateTime64InJodaSyntax('10:30:50', 'HH:mm:ss'); +select parseDateTime64InJodaSyntax('2024-12-31 23:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('2024-01-01 00:00:01.123456+0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); -- { serverError CANNOT_PARSE_DATETIME } select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('2024-11-05-0800 01:02:03.123456', 'yyyy-MM-ddZ HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); - +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456Australia/Adelaide', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntax('999999 10-09-202410:30:10', 'SSSSSSSSS dd-MM-yyyyHH:mm:ss'); +select parseDateTime64InJodaSyntax('2024-10-09 10:30:10.123456-0845', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); +select parseDateTime64InJodaSyntax('2023-02-29 11:22:33Not/Timezone', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError BAD_ARGUMENTS } +--leap years and non-leap years +select parseDateTime64InJodaSyntax('2024-02-29 11:23:34America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); -- { serverError CANNOT_PARSE_DATETIME } +select parseDateTime64InJodaSyntax('2024-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2023-02-28 23:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntax('2024-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); +select parseDateTime64InJodaSyntax('2023-03-01 00:22:33-8000', 'yyyy-MM-dd HH:mm:ssZ'); +select 'parseDateTime64InJodaSyntaxOrZero'; select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); - +select parseDateTime64InJodaSyntaxOrZero('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrZero('wrong value', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrZero('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntaxOrZero('', ''); +select parseDateTime64InJodaSyntaxOrZero('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); +select 'parseDateTime64InJodaSyntaxOrNull'; select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456', 'yyyy-MM-dd HH:mm:ss.SSSSSS'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456789', 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456-0800', 'yyyy-MM-dd HH:mm:ss.SSSSSSZ'); select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123456America/Los_Angeles', 'yyyy-MM-dd HH:mm:ss.SSSSSSz'); +select parseDateTime64InJodaSyntaxOrNull('2024-10-09 10:30:10.123', 'yyyy-dd-MM HH:mm:ss.SSS'); +select parseDateTime64InJodaSyntaxOrNull('2023-02-29 11:22:33America/Los_Angeles', 'yyyy-MM-dd HH:mm:ssz'); +select parseDateTime64InJodaSyntaxOrNull('', ''); +select parseDateTime64InJodaSyntaxOrNull('2177-10-09 10:30:10.123', 'yyyy-MM-dd HH:mm:ss.SSS'); + +set session_timezone = 'UTC'; +select parseDateTime64InJodaSyntax('', ''); diff --git a/tests/queries/0_stateless/03256_invalid_mutation_query.reference b/tests/queries/0_stateless/03256_invalid_mutation_query.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03256_invalid_mutation_query.sql b/tests/queries/0_stateless/03256_invalid_mutation_query.sql new file mode 100644 index 00000000000..9b4e8f9a7ea --- /dev/null +++ b/tests/queries/0_stateless/03256_invalid_mutation_query.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t (x int) ENGINE = MergeTree() ORDER BY (); + +DELETE FROM t WHERE y in (SELECT x FROM t); -- { serverError UNKNOWN_IDENTIFIER } +DELETE FROM t WHERE x in (SELECT y FROM t); -- { serverError UNKNOWN_IDENTIFIER } +DELETE FROM t WHERE x IN (SELECT * FROM t2); -- { serverError UNKNOWN_TABLE } +ALTER TABLE t DELETE WHERE x in (SELECT y FROM t); -- { serverError UNKNOWN_IDENTIFIER } +ALTER TABLE t UPDATE x = 1 WHERE x IN (SELECT y FROM t); -- { serverError UNKNOWN_IDENTIFIER } + +DELETE FROM t WHERE x IN (SELECT foo FROM bar) SETTINGS validate_mutation_query = 0; + +ALTER TABLE t ADD COLUMN y int; +DELETE FROM t WHERE y in (SELECT y FROM t); + +CREATE TABLE t2 (x int) ENGINE = MergeTree() ORDER BY (); +DELETE FROM t WHERE x IN (SELECT * FROM t2); + +DROP TABLE t; +DROP TABLE t2; diff --git a/tests/queries/0_stateless/03259_orc_date_out_of_range.reference b/tests/queries/0_stateless/03259_orc_date_out_of_range.reference new file mode 100644 index 00000000000..ddac785369f --- /dev/null +++ b/tests/queries/0_stateless/03259_orc_date_out_of_range.reference @@ -0,0 +1,12 @@ +number Nullable(Int64) +date_field Nullable(Date32) +\N +1970-01-02 +\N +1970-01-04 +\N +1970-01-06 +\N +1970-01-08 +\N +1970-01-10 diff --git a/tests/queries/0_stateless/03259_orc_date_out_of_range.sql b/tests/queries/0_stateless/03259_orc_date_out_of_range.sql new file mode 100644 index 00000000000..e73d2faa5dd --- /dev/null +++ b/tests/queries/0_stateless/03259_orc_date_out_of_range.sql @@ -0,0 +1,14 @@ +-- Tags: no-fasttest, no-parallel + +SET session_timezone = 'UTC'; +SET engine_file_truncate_on_insert = 1; + +insert into function file('03259.orc', 'ORC') +select + number, + if (number % 2 = 0, null, toDate32(number)) as date_field +from numbers(10); + +desc file('03259.orc', 'ORC'); + +select date_field from file('03259.orc', 'ORC') order by number; diff --git a/tests/queries/0_stateless/03260_compressor_stat.reference b/tests/queries/0_stateless/03260_compressor_stat.reference new file mode 100644 index 00000000000..ba84b26cc48 --- /dev/null +++ b/tests/queries/0_stateless/03260_compressor_stat.reference @@ -0,0 +1 @@ +CODEC(Delta(1), LZ4) 14 48 diff --git a/tests/queries/0_stateless/03260_compressor_stat.sh b/tests/queries/0_stateless/03260_compressor_stat.sh new file mode 100755 index 00000000000..8a03541763c --- /dev/null +++ b/tests/queries/0_stateless/03260_compressor_stat.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo "Hello, World!" | $CLICKHOUSE_COMPRESSOR --codec 'Delta' --codec 'LZ4' | $CLICKHOUSE_COMPRESSOR --stat diff --git a/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.reference b/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.sql b/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.sql new file mode 100644 index 00000000000..e0018632be4 --- /dev/null +++ b/tests/queries/0_stateless/03261_optimize_rewrite_array_exists_to_has_crash.sql @@ -0,0 +1,10 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/71382 +DROP TABLE IF EXISTS rewrite; +CREATE TABLE rewrite (c0 Int) ENGINE = Memory(); +SELECT 1 +FROM rewrite +INNER JOIN rewrite AS y ON ( + SELECT 1 +) +INNER JOIN rewrite AS z ON 1 +SETTINGS optimize_rewrite_array_exists_to_has=1; diff --git a/tests/queries/0_stateless/03262_column_sizes_with_dynamic_structure.reference b/tests/queries/0_stateless/03262_column_sizes_with_dynamic_structure.reference new file mode 100644 index 00000000000..5e30d36d620 --- /dev/null +++ b/tests/queries/0_stateless/03262_column_sizes_with_dynamic_structure.reference @@ -0,0 +1 @@ +test 10000000 352 39 39 diff --git a/tests/queries/0_stateless/03262_column_sizes_with_dynamic_structure.sql b/tests/queries/0_stateless/03262_column_sizes_with_dynamic_structure.sql new file mode 100644 index 00000000000..931990d582f --- /dev/null +++ b/tests/queries/0_stateless/03262_column_sizes_with_dynamic_structure.sql @@ -0,0 +1,23 @@ +-- Tags: no-random-settings, no-fasttest + +set allow_experimental_dynamic_type = 1; +set allow_experimental_json_type = 1; + + +drop table if exists test; +create table test (d Dynamic, json JSON) engine=MergeTree order by tuple() settings min_rows_for_wide_part=0, min_bytes_for_wide_part=1; +insert into test select number, '{"a" : 42, "b" : "Hello, World"}' from numbers(10000000); + +SELECT + `table`, + sum(rows) AS rows, + floor(sum(data_uncompressed_bytes) / (1024 * 1024)) AS data_size_uncompressed, + floor(sum(data_compressed_bytes) / (1024 * 1024)) AS data_size_compressed, + floor(sum(bytes_on_disk) / (1024 * 1024)) AS total_size_on_disk +FROM system.parts +WHERE active AND (database = currentDatabase()) AND (`table` = 'test') +GROUP BY `table` +ORDER BY `table` ASC; + +drop table test; + diff --git a/tests/queries/0_stateless/03262_filter_push_down_view.reference b/tests/queries/0_stateless/03262_filter_push_down_view.reference new file mode 100644 index 00000000000..5c78173bb5a --- /dev/null +++ b/tests/queries/0_stateless/03262_filter_push_down_view.reference @@ -0,0 +1,2 @@ +Condition: (_CAST(toDate(ts)) in (-Inf, 1703980800]) +Granules: 3/3 diff --git a/tests/queries/0_stateless/03262_filter_push_down_view.sql b/tests/queries/0_stateless/03262_filter_push_down_view.sql new file mode 100644 index 00000000000..8492d8c8ebd --- /dev/null +++ b/tests/queries/0_stateless/03262_filter_push_down_view.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS alpha; +DROP TABLE IF EXISTS alpha__day; + +SET session_timezone = 'Etc/UTC'; + +CREATE TABLE alpha +( + `ts` DateTime64(6), + `auid` Int64, +) +ENGINE = MergeTree +ORDER BY (auid, ts) +SETTINGS index_granularity = 1; + +CREATE VIEW alpha__day +( + `ts_date` Date, + `auid` Int64, +) +AS SELECT + ts_date, + auid, +FROM +( + SELECT + toDate(ts) AS ts_date, + auid + FROM alpha +) +WHERE ts_date <= toDateTime('2024-01-01 00:00:00') - INTERVAL 1 DAY; + +INSERT INTO alpha VALUES (toDateTime64('2024-01-01 00:00:00.000', 3) - INTERVAL 3 DAY, 1); +INSERT INTO alpha VALUES (toDateTime64('2024-01-01 00:00:00.000', 3) - INTERVAL 3 DAY, 2); +INSERT INTO alpha VALUES (toDateTime64('2024-01-01 00:00:00.000', 3) - INTERVAL 3 DAY, 3); + +select trimLeft(explain) from (EXPLAIN indexes = 1 SELECT auid FROM alpha__day WHERE auid = 1) where explain like '%Condition:%' or explain like '%Granules:%' settings allow_experimental_analyzer = 1; diff --git a/tests/queries/0_stateless/03268_empty_tuple_update.reference b/tests/queries/0_stateless/03268_empty_tuple_update.reference new file mode 100644 index 00000000000..30bc45d7a18 --- /dev/null +++ b/tests/queries/0_stateless/03268_empty_tuple_update.reference @@ -0,0 +1 @@ +() 2 diff --git a/tests/queries/0_stateless/03268_empty_tuple_update.sql b/tests/queries/0_stateless/03268_empty_tuple_update.sql new file mode 100644 index 00000000000..343117719fc --- /dev/null +++ b/tests/queries/0_stateless/03268_empty_tuple_update.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t0; + +CREATE TABLE t0 (c0 Tuple(), c1 int) ENGINE = Memory(); + +INSERT INTO t0 VALUES ((), 1); + +ALTER TABLE t0 UPDATE c0 = (), c1 = 2 WHERE EXISTS (SELECT 1); + +SELECT * FROM t0; + +DROP TABLE t0; diff --git a/tests/queries/0_stateless/03270_processors_profile_log_3.reference b/tests/queries/0_stateless/03270_processors_profile_log_3.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/03270_processors_profile_log_3.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/03270_processors_profile_log_3.sh b/tests/queries/0_stateless/03270_processors_profile_log_3.sh new file mode 100755 index 00000000000..eb86a9f6352 --- /dev/null +++ b/tests/queries/0_stateless/03270_processors_profile_log_3.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t + ( + a UInt32, + b UInt32 + ) + ENGINE = MergeTree + ORDER BY (a, b); + + INSERT INTO t SELECT number, number FROM numbers(1000); +" + +query_id="03270_processors_profile_log_3_$RANDOM" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SET log_processors_profiles = 1; + + WITH + t0 AS + ( + SELECT * + FROM numbers(1000) + ), + t1 AS + ( + SELECT number * 3 AS b + FROM t0 + ) + SELECT b * 3 + FROM t + WHERE a IN (t1) + FORMAT Null; +" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SYSTEM FLUSH LOGS; + + SELECT sum(elapsed_us) > 0 + FROM system.processors_profile_log + WHERE event_date >= yesterday() AND query_id = '$query_id' AND name = 'CreatingSetsTransform'; +" + +##################################################################### + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t1 + ( + st FixedString(54) + ) + ENGINE = MergeTree + ORDER BY tuple(); + + INSERT INTO t1 VALUES + ('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRTUVWXYZ'), + ('\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'), + ('IIIIIIIIII\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'); +" + +query_id="03270_processors_profile_log_3_$RANDOM" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SET log_processors_profiles = 1; + SET max_threads=2; -- no merging when max_threads=1 + + WITH + ( + SELECT groupConcat(',')(st) + FROM t1 + ORDER BY ALL + ) AS a, + ( + SELECT groupConcat(',')(CAST(st, 'String')) + FROM t1 + ORDER BY ALL + ) AS b + SELECT a = b + FORMAT Null; +" + +$CLICKHOUSE_CLIENT --query_id="$query_id" -q " + SYSTEM FLUSH LOGS; + + SELECT sum(elapsed_us) > 0 + FROM system.processors_profile_log + WHERE event_date >= yesterday() AND query_id = '$query_id' AND name = 'MergingSortedTransform'; +" + diff --git a/tests/queries/0_stateless/03271_parse_sparse_columns_defaults.reference b/tests/queries/0_stateless/03271_parse_sparse_columns_defaults.reference new file mode 100644 index 00000000000..c76adef1c23 --- /dev/null +++ b/tests/queries/0_stateless/03271_parse_sparse_columns_defaults.reference @@ -0,0 +1,4 @@ +1 false +2 false +all_1_1_0 Sparse +all_2_2_0 Sparse diff --git a/tests/queries/0_stateless/03271_parse_sparse_columns_defaults.sh b/tests/queries/0_stateless/03271_parse_sparse_columns_defaults.sh new file mode 100755 index 00000000000..aa80de03754 --- /dev/null +++ b/tests/queries/0_stateless/03271_parse_sparse_columns_defaults.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS test_default_bool; + + CREATE TABLE test_default_bool (id Int8, b Bool DEFAULT false) + ENGINE = MergeTree ORDER BY id + SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; +" + +echo 'INSERT INTO test_default_bool FORMAT CSV 1,\N' | $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" --data-binary @- +echo 'INSERT INTO test_default_bool FORMAT CSV 2,\N' | $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" --data-binary @- + +$CLICKHOUSE_CLIENT --query " + SELECT * FROM test_default_bool ORDER BY id; + SELECT name, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 'test_default_bool' AND column = 'b' AND active; + DROP TABLE test_default_bool; +" diff --git a/tests/queries/0_stateless/03271_sqllancer_having_issue.reference b/tests/queries/0_stateless/03271_sqllancer_having_issue.reference new file mode 100644 index 00000000000..2ab0d6f120f --- /dev/null +++ b/tests/queries/0_stateless/03271_sqllancer_having_issue.reference @@ -0,0 +1,2 @@ +-0.07947094746692918 -1017248723 0 +-0.07947094746692918 -1017248723 0 diff --git a/tests/queries/0_stateless/03271_sqllancer_having_issue.sql b/tests/queries/0_stateless/03271_sqllancer_having_issue.sql new file mode 100644 index 00000000000..aa9659693f0 --- /dev/null +++ b/tests/queries/0_stateless/03271_sqllancer_having_issue.sql @@ -0,0 +1,10 @@ +-- https://s3.amazonaws.com/clickhouse-test-reports/0/a02b20a9813c6ba0880c67f079363ef1c5440109/sqlancer__debug_.html +-- Caused by enablement of query_plan_merge_filters. Will fail if the next line is uncommented +-- set query_plan_merge_filters=1; + +CREATE TABLE IF NOT EXISTS t3 (c0 Int32) ENGINE = Memory() ; +INSERT INTO t3(c0) VALUES (1110866669); + +-- These 2 queries are expected to return the same +SELECT (tan (t3.c0)), SUM(-1017248723), ((t3.c0)%(t3.c0)) FROM t3 GROUP BY t3.c0 SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0; +SELECT (tan (t3.c0)), SUM(-1017248723), ((t3.c0)%(t3.c0)) FROM t3 GROUP BY t3.c0 HAVING ((tan ((- (SUM(-1017248723)))))) and ((sqrt (SUM(-1017248723)))) UNION ALL SELECT (tan (t3.c0)), SUM(-1017248723), ((t3.c0)%(t3.c0)) FROM t3 GROUP BY t3.c0 HAVING (NOT (((tan ((- (SUM(-1017248723)))))) and ((sqrt (SUM(-1017248723)))))) UNION ALL SELECT (tan (t3.c0)), SUM(-1017248723), ((t3.c0)%(t3.c0)) FROM t3 GROUP BY t3.c0 HAVING ((((tan ((- (SUM(-1017248723)))))) and ((sqrt (SUM(-1017248723))))) IS NULL) SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0;