From 90b7ae871ed9e4e418eec37f5ccfcd25ecae264e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Nov 2024 22:23:05 +0000 Subject: [PATCH 01/38] Add inferred format name to create query in File/S3/URL/HDFS/Azure engines --- src/Interpreters/InterpreterCreateQuery.cpp | 21 ++++- src/Storages/IStorage.h | 4 + .../ObjectStorage/Azure/Configuration.cpp | 54 ++++++++----- .../ObjectStorage/Azure/Configuration.h | 3 +- .../ObjectStorage/HDFS/Configuration.cpp | 16 ++-- .../ObjectStorage/HDFS/Configuration.h | 3 +- .../ObjectStorage/Local/Configuration.h | 2 +- .../ObjectStorage/S3/Configuration.cpp | 55 +++++++------ src/Storages/ObjectStorage/S3/Configuration.h | 3 +- .../ObjectStorage/StorageObjectStorage.cpp | 5 ++ .../ObjectStorage/StorageObjectStorage.h | 4 +- .../StorageObjectStorageCluster.cpp | 2 +- src/Storages/StorageFile.cpp | 5 ++ src/Storages/StorageFile.h | 2 + src/Storages/StorageURL.cpp | 7 ++ src/Storages/StorageURL.h | 2 + src/TableFunctions/ITableFunctionCluster.h | 2 +- src/TableFunctions/ITableFunctionFileLike.cpp | 12 +-- src/TableFunctions/ITableFunctionFileLike.h | 2 +- .../TableFunctionObjectStorage.h | 2 +- src/TableFunctions/TableFunctionURL.cpp | 6 +- src/TableFunctions/TableFunctionURL.h | 2 +- .../test_storage_azure_blob_storage/test.py | 79 +++++++++++++++++++ tests/integration/test_storage_hdfs/test.py | 28 ++++++- ...at_inference_create_query_s3_url.reference | 14 ++++ ...3_format_inference_create_query_s3_url.sql | 57 +++++++++++++ ...rmat_inference_create_query_file.reference | 1 + ...3274_format_inference_create_query_file.sh | 19 +++++ .../queries/0_stateless/data_minio/json_data | 1 + 29 files changed, 345 insertions(+), 68 deletions(-) create mode 100644 tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference create mode 100644 tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql create mode 100644 tests/queries/0_stateless/03274_format_inference_create_query_file.reference create mode 100755 tests/queries/0_stateless/03274_format_inference_create_query_file.sh create mode 100644 tests/queries/0_stateless/data_minio/json_data diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f6586f8bfc2..67c4d2c0413 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1181,6 +1181,22 @@ namespace source_ast->children.push_back(source_ast->elements); dict.set(dict.source, source_ast); } + + ASTs * getEngineArgsFromCreateQuery(ASTCreateQuery & create_query) + { + ASTStorage * storage_def = create_query.storage; + if (!storage_def) + return nullptr; + + if (!storage_def->engine) + return nullptr; + + const ASTFunction & engine_def = *storage_def->engine; + if (!engine_def.arguments) + return nullptr; + + return &engine_def.arguments->children; + } } void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const @@ -1870,7 +1886,10 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, mode); /// If schema wes inferred while storage creation, add columns description to create query. - addColumnsDescriptionToCreateQueryIfNecessary(query_ptr->as(), res); + auto & create_query = query_ptr->as(); + addColumnsDescriptionToCreateQueryIfNecessary(create_query, res); + if (auto * engine_args = getEngineArgsFromCreateQuery(create_query)) + res->updateEngineArgsForCreateQuery(*engine_args, getContext()); } validateVirtualColumns(*res); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 0dc48634282..e99a66b9bda 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -284,6 +284,10 @@ public: /// Returns hints for serialization of columns accorsing to statistics accumulated by storage. virtual SerializationInfoByName getSerializationHints() const { return {}; } + /// Update engine args in create query that were inferred during storage creation to avoid the same + /// inference on server restart. For example - data format inference in File/URL/S3/etc engines. + virtual void updateEngineArgsForCreateQuery(ASTs & /*args*/, const ContextPtr & /*context*/) const {} + private: StorageID storage_id; diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index d08e0d9debc..faa1554dbe8 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -283,7 +283,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, } void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( - ASTs & args, const String & structure_, const String & format_, ContextPtr context) + ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -295,7 +295,7 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -319,9 +319,12 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( if (args.size() == 3) { args.push_back(format_literal); - /// Add compression = "auto" before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression = "auto" before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } /// (connection_string, container_name, blobpath, structure) or /// (connection_string, container_name, blobpath, format) @@ -334,12 +337,15 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (fourth_arg == "auto") args[3] = format_literal; - /// Add compression=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression=auto before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } /// (..., structure) -> (..., format, compression, structure) - else + else if (with_structure) { auto structure_arg = args.back(); args[3] = format_literal; @@ -362,15 +368,19 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (fourth_arg == "auto") args[3] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// (..., account_name, account_key) -> (..., account_name, account_key, format, compression, structure) else { args.push_back(format_literal); - /// Add compression=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression=auto before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } } /// (connection_string, container_name, blobpath, format, compression, structure) or @@ -386,7 +396,7 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (fourth_arg == "auto") args[3] = format_literal; - if (checkAndGetLiteralArgument(args[5], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[5], "structure") == "auto") args[5] = structure_literal; } /// (..., account_name, account_key, format) -> (..., account_name, account_key, format, compression, structure) @@ -394,12 +404,15 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( { if (sixth_arg == "auto") args[5] = format_literal; - /// Add compression=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + if (with_structure) + { + /// Add compression=auto before structure argument. + args.push_back(std::make_shared("auto")); + args.push_back(structure_literal); + } } /// (..., account_name, account_key, structure) -> (..., account_name, account_key, format, compression, structure) - else + else if (with_structure) { auto structure_arg = args.back(); args[5] = format_literal; @@ -417,14 +430,15 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( /// (..., format, compression) -> (..., format, compression, structure) if (checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// (storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure) else if (args.size() == 8) { if (checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = format_literal; - if (checkAndGetLiteralArgument(args[7], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[7], "structure") == "auto") args[7] = structure_literal; } } diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 21db81802c7..72124465c46 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -76,7 +76,8 @@ public: ASTs & args, const String & structure_, const String & format_, - ContextPtr context) override; + ContextPtr context, + bool with_structure) override; protected: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 6bee4154b2f..3c4897b5062 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -174,7 +174,8 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, - ContextPtr context) + ContextPtr context, + bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -186,7 +187,7 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -209,23 +210,26 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( if (count == 1) { /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + args.push_back(format_literal); + if (with_structure) + args.push_back(structure_literal); } /// hdfs(url, format) else if (count == 2) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args.back() = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// hdfs(url, format, structure) /// hdfs(url, format, structure, compression_method) + /// hdfs(url, format, compression_method) else if (count >= 3) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } } diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 90997292693..db8ab7f9e4d 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -62,7 +62,8 @@ public: ASTs & args, const String & structure_, const String & format_, - ContextPtr context) override; + ContextPtr context, + bool with_structure) override; private: void fromNamedCollection(const NamedCollection &, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index 32a095bf7de..d679b4ad724 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -59,7 +59,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared("/"); } - void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { } + void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr, bool) override { } private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 02effe261d0..629628c762f 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -395,7 +395,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( - ASTs & args, const String & structure_, const String & format_, ContextPtr context) + ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -407,7 +407,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -429,8 +429,9 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( if (count == 1) { /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); - args.push_back(structure_literal); + args.push_back(format_literal); + if (with_structure) + args.push_back(structure_literal); } /// s3(s3_url, format) or /// s3(s3_url, NOSIGN) @@ -444,11 +445,13 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( else if (checkAndGetLiteralArgument(args[1], "format") == "auto") args[1] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// s3(source, format, structure) or /// s3(source, access_key_id, secret_access_key) or - /// s3(source, NOSIGN, format) + /// s3(source, NOSIGN, format) or + /// s3(source, format, compression_method) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither. else if (count == 3) { @@ -457,26 +460,29 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { if (second_arg == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } else { /// Add format and structure arguments. args.push_back(format_literal); - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } } /// s3(source, format, structure, compression_method) or /// s3(source, access_key_id, secret_access_key, format) or /// s3(source, access_key_id, secret_access_key, session_token) or - /// s3(source, NOSIGN, format, structure) + /// s3(source, NOSIGN, format, structure) or + /// s3(source, NOSIGN, format, compression_method) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither. else if (count == 4) { @@ -485,14 +491,14 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; - if (checkAndGetLiteralArgument(args[3], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[3], "structure") == "auto") args[3] = structure_literal; } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { if (second_arg == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } else @@ -502,18 +508,21 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } else { args.push_back(format_literal); - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } } } /// s3(source, access_key_id, secret_access_key, format, structure) or /// s3(source, access_key_id, secret_access_key, session_token, format) or - /// s3(source, NOSIGN, format, structure, compression_method) + /// s3(source, NOSIGN, format, structure, compression_method) or + /// s3(source, access_key_id, secret_access_key, format, compression) /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not. else if (count == 5) { @@ -522,7 +531,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[3] = structure_literal; } else @@ -532,19 +541,21 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; - if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[4], "structure") == "auto") args[4] = structure_literal; } else { if (checkAndGetLiteralArgument(args[4], "format") == "auto") args[4] = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } } } /// s3(source, access_key_id, secret_access_key, format, structure, compression) or - /// s3(source, access_key_id, secret_access_key, session_token, format, structure) + /// s3(source, access_key_id, secret_access_key, session_token, format, structure) or + /// s3(source, access_key_id, secret_access_key, session_token, format, compression_method) else if (count == 6) { auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); @@ -552,14 +563,14 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; - if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[4], "structure") == "auto") args[4] = structure_literal; } else { if (checkAndGetLiteralArgument(args[4], "format") == "auto") args[4] = format_literal; - if (checkAndGetLiteralArgument(args[5], "format") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = structure_literal; } } @@ -568,7 +579,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( { if (checkAndGetLiteralArgument(args[4], "format") == "auto") args[4] = format_literal; - if (checkAndGetLiteralArgument(args[5], "format") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[5], "format") == "auto") args[5] = structure_literal; } } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 87f2be1bf3e..c4614a28189 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -91,7 +91,8 @@ public: ASTs & args, const String & structure, const String & format, - ContextPtr context) override; + ContextPtr context, + bool with_structure) override; private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index fd2fe0400bb..419069dbb8d 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -479,6 +479,11 @@ std::pair StorageObjectStorage::resolveSchemaAn return std::pair(columns, format); } +void StorageObjectStorage::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +{ + configuration->addStructureAndFormatToArgsIfNeeded(args, "", configuration->format, context, /*with_structure=*/false); +} + SchemaCache & StorageObjectStorage::getSchemaCache(const ContextPtr & context, const std::string & storage_type_name) { if (storage_type_name == "s3") diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index e2bb41a4935..81f9828ccc8 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -122,6 +122,8 @@ public: std::string & sample_path, const ContextPtr & context); + void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + protected: String getPathSample(StorageInMemoryMetadata metadata, ContextPtr context); @@ -179,7 +181,7 @@ public: /// Add/replace structure and format arguments in the AST arguments if they have 'auto' values. virtual void addStructureAndFormatToArgsIfNeeded( - ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0; + ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) = 0; bool withPartitionWildcard() const; bool withGlobs() const { return isPathWithGlobs() || isNamespaceWithGlobs(); } diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 176f20cf5ca..07eecc65599 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -107,7 +107,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context); + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index eefd60128a6..91263180b5c 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -2114,6 +2114,11 @@ void StorageFile::truncate( } } +void StorageFile::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +{ + if (checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context), "format") == "auto") + args[0] = std::make_shared(format_name); +} void registerStorageFile(StorageFactory & factory) { diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 6b21353f161..5b95d1a9f8b 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -139,6 +139,8 @@ public: bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } + void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + protected: friend class StorageFileSource; friend class StorageFileSink; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 3ba8d1fa304..af91768f79e 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -38,6 +38,8 @@ #include #include +#include + #include #include #include @@ -1570,6 +1572,11 @@ void StorageURL::processNamedCollectionResult(Configuration & configuration, con configuration.structure = collection.getOrDefault("structure", "auto"); } +void StorageURL::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +{ + TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(args, "", format_name, context, /*with_structure=*/false); +} + StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, const ContextPtr & local_context) { StorageURL::Configuration configuration; diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 6f1d544629a..4f3e920afac 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -305,6 +305,8 @@ public: bool supportsDynamicSubcolumns() const override { return true; } + void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); struct Configuration : public StatelessTableEngineConfiguration diff --git a/src/TableFunctions/ITableFunctionCluster.h b/src/TableFunctions/ITableFunctionCluster.h index 744d7139d16..3378cbe87d3 100644 --- a/src/TableFunctions/ITableFunctionCluster.h +++ b/src/TableFunctions/ITableFunctionCluster.h @@ -31,7 +31,7 @@ public: ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - Base::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context); + Base::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, /*with_structure=*/true); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 23e59494f61..b591ea97e3d 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -88,7 +88,7 @@ void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & compression_method = checkAndGetLiteralArgument(args[3], "compression_method"); } -void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context) +void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context, bool with_structure) { if (args.empty() || args.size() > getMaxNumberOfArguments()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); @@ -103,21 +103,23 @@ void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & ar if (args.size() == 1) { args.push_back(format_literal); - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } /// f(filename, format) else if (args.size() == 2) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args.back() = format_literal; - args.push_back(structure_literal); + if (with_structure) + args.push_back(structure_literal); } - /// f(filename, format, structure) or f(filename, format, structure, compression) + /// f(filename, format, structure) or f(filename, format, structure, compression) or f(filename, format, compression) else if (args.size() >= 3) { if (checkAndGetLiteralArgument(args[1], "format") == "auto") args[1] = format_literal; - if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") args[2] = structure_literal; } } diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 4c97507b8d1..f9b5e30e85e 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -35,7 +35,7 @@ public: static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure); protected: diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 19cd637bd80..855acca85ee 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -139,7 +139,7 @@ public: const String & format, const ContextPtr & context) { - Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context); + Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context, /*with_structure=*/true); } protected: diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 8f4841a992b..5de3a302772 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -77,7 +77,7 @@ void TableFunctionURL::parseArgumentsImpl(ASTs & args, const ContextPtr & contex } } -void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context) +void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure) { if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { @@ -89,7 +89,7 @@ void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, co auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); args.push_back(format_equal_func); } - if (collection->getOrDefault("structure", "auto") == "auto") + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") { ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); @@ -109,7 +109,7 @@ void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, co args.pop_back(); } - ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context); + ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, with_structure); if (headers_ast) args.push_back(headers_ast); diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index a1efddb84c6..d2c9d4d9ddf 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -34,7 +34,7 @@ public: ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure); protected: void parseArguments(const ASTPtr & ast, ContextPtr context) override; diff --git a/tests/integration/test_storage_azure_blob_storage/test.py b/tests/integration/test_storage_azure_blob_storage/test.py index 220ee13cb25..e22ce925f6c 100644 --- a/tests/integration/test_storage_azure_blob_storage/test.py +++ b/tests/integration/test_storage_azure_blob_storage/test.py @@ -1314,6 +1314,7 @@ def test_size_virtual_column(cluster): def test_format_detection(cluster): node = cluster.instances["node"] + connection_string = cluster.env_variables["AZURITE_CONNECTION_STRING"] storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] account_name = "devstoreaccount1" account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" @@ -1381,6 +1382,84 @@ def test_format_detection(cluster): assert result == expected_result + azure_query( + node, + f"create table test_format_detection engine=AzureBlobStorage('{connection_string}', 'cont', 'test_format_detection1')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{connection_string}\\', \\'cont\\', \\'test_format_detection1\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{connection_string}', 'cont', 'test_format_detection1', auto)", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{connection_string}\\', \\'cont\\', \\'test_format_detection1\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{connection_string}', 'cont', 'test_format_detection1', auto, 'none')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{connection_string}\\', \\'cont\\', \\'test_format_detection1\\', \\'JSON\\', \\'none\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{storage_account_url}', 'cont', 'test_format_detection1', '{account_name}', '{account_key}')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{storage_account_url}\\', \\'cont\\', \\'test_format_detection1\\', \\'{account_name}\\', \\'{account_key}\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{storage_account_url}', 'cont', 'test_format_detection1', '{account_name}', '{account_key}', auto)", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{storage_account_url}\\', \\'cont\\', \\'test_format_detection1\\', \\'{account_name}\\', \\'{account_key}\\', \\'JSON\\')\n" + ) + + azure_query( + node, + f"create or replace table test_format_detection engine=AzureBlobStorage('{storage_account_url}', 'cont', 'test_format_detection1', '{account_name}', '{account_key}', auto, 'none')", + ) + result = azure_query( + node, + f"show create table test_format_detection", + ) + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = AzureBlobStorage(\\'{storage_account_url}\\', \\'cont\\', \\'test_format_detection1\\', \\'{account_name}\\', \\'{account_key}\\', \\'JSON\\', \\'none\\')\n" + ) + def test_write_to_globbed_partitioned_path(cluster): node = cluster.instances["node"] diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 366bc28d2c9..9f0afc57ad9 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -636,7 +636,7 @@ def test_multiple_inserts(started_cluster): node1.query(f"drop table test_multiple_inserts") -def test_format_detection(started_cluster): +def test_format_detection_from_file_name(started_cluster): node1.query( f"create table arrow_table (x UInt64) engine=HDFS('hdfs://hdfs1:9000/data.arrow')" ) @@ -1222,6 +1222,32 @@ def test_format_detection(started_cluster): assert expected_result == result + node.query( + f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1')" + ) + result = node.query( + f"show create table test_format_detection" + ) + assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" + + node.query("drop table test_format_detection") + node.query( + f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto)" + ) + result = node.query( + f"show create table test_format_detection" + ) + assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" + + node.query("drop table test_format_detection") + node.query( + f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto, 'none')" + ) + result = node.query( + f"show create table test_format_detection" + ) + assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\', \\'none\\')\n" + def test_write_to_globbed_partitioned_path(started_cluster): node = started_cluster.instances["node1"] diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference new file mode 100644 index 00000000000..72696cef342 --- /dev/null +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.reference @@ -0,0 +1,14 @@ +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'NOSIGN\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = S3(\'http://localhost:11111/test/json_data\', \'test\', \'[HIDDEN]\', \'\', \'JSON\', \'none\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\') +CREATE TABLE default.test\n(\n `a` Nullable(Int64)\n)\nENGINE = URL(\'http://localhost:11111/test/json_data\', \'JSON\', \'none\') diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql new file mode 100644 index 00000000000..353b78b1e15 --- /dev/null +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql @@ -0,0 +1,57 @@ +drop table if exists test; + +create table test engine=S3('http://localhost:11111/test/json_data'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', NOSIGN); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', auto); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', auto, 'none'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', NOSIGN, auto); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', NOSIGN, auto, 'none'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', ''); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', '', auto); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', auto, 'none'); +show create table test; +drop table test; + +create table test engine=S3('http://localhost:11111/test/json_data', 'test', 'testtest', '', auto, 'none'); +show create table test; +drop table test; + +create table test engine=URL('http://localhost:11111/test/json_data'); +show create table test; +drop table test; + +create table test engine=URL('http://localhost:11111/test/json_data', auto); +show create table test; +drop table test; + +create table test engine=URL('http://localhost:11111/test/json_data', auto, 'none'); +show create table test; +drop table test; diff --git a/tests/queries/0_stateless/03274_format_inference_create_query_file.reference b/tests/queries/0_stateless/03274_format_inference_create_query_file.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/03274_format_inference_create_query_file.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/03274_format_inference_create_query_file.sh b/tests/queries/0_stateless/03274_format_inference_create_query_file.sh new file mode 100755 index 00000000000..10c0650144c --- /dev/null +++ b/tests/queries/0_stateless/03274_format_inference_create_query_file.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select 42 as a format JSONEachRow" > data_$CLICKHOUSE_TEST_UNIQUE_NAME +$CLICKHOUSE_LOCAL -nm -q " +create table test engine=File(auto, './data_$CLICKHOUSE_TEST_UNIQUE_NAME'); +show create table test; +drop table test; + +create table test engine=File(auto, './data_$CLICKHOUSE_TEST_UNIQUE_NAME', 'none'); +show create table test; +drop table test; +" | grep "JSON" -c + +rm data_$CLICKHOUSE_TEST_UNIQUE_NAME + diff --git a/tests/queries/0_stateless/data_minio/json_data b/tests/queries/0_stateless/data_minio/json_data new file mode 100644 index 00000000000..bfe33aeb603 --- /dev/null +++ b/tests/queries/0_stateless/data_minio/json_data @@ -0,0 +1 @@ +{"a" : 42} From 1fc4757c0829a9b013972858c2eddb293bd3cce8 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Nov 2024 22:27:29 +0000 Subject: [PATCH 02/38] Better naming --- src/Interpreters/InterpreterCreateQuery.cpp | 3 ++- src/Storages/IStorage.h | 4 ++-- src/Storages/ObjectStorage/StorageObjectStorage.cpp | 2 +- src/Storages/ObjectStorage/StorageObjectStorage.h | 2 +- src/Storages/StorageFile.cpp | 2 +- src/Storages/StorageFile.h | 2 +- src/Storages/StorageURL.cpp | 2 +- src/Storages/StorageURL.h | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 67c4d2c0413..a8cbf83ff23 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1888,8 +1888,9 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, /// If schema wes inferred while storage creation, add columns description to create query. auto & create_query = query_ptr->as(); addColumnsDescriptionToCreateQueryIfNecessary(create_query, res); + /// Add any inferred engine args if needed. For example, data format for engines File/S3/URL/etc if (auto * engine_args = getEngineArgsFromCreateQuery(create_query)) - res->updateEngineArgsForCreateQuery(*engine_args, getContext()); + res->addInferredEngineArgsToCreateQuery(*engine_args, getContext()); } validateVirtualColumns(*res); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index e99a66b9bda..37359aad290 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -284,9 +284,9 @@ public: /// Returns hints for serialization of columns accorsing to statistics accumulated by storage. virtual SerializationInfoByName getSerializationHints() const { return {}; } - /// Update engine args in create query that were inferred during storage creation to avoid the same + /// Add engine args that were inferred during storage creation to create query to avoid the same /// inference on server restart. For example - data format inference in File/URL/S3/etc engines. - virtual void updateEngineArgsForCreateQuery(ASTs & /*args*/, const ContextPtr & /*context*/) const {} + virtual void addInferredEngineArgsToCreateQuery(ASTs & /*args*/, const ContextPtr & /*context*/) const {} private: StorageID storage_id; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 419069dbb8d..098a5549fae 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -479,7 +479,7 @@ std::pair StorageObjectStorage::resolveSchemaAn return std::pair(columns, format); } -void StorageObjectStorage::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +void StorageObjectStorage::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const { configuration->addStructureAndFormatToArgsIfNeeded(args, "", configuration->format, context, /*with_structure=*/false); } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 81f9828ccc8..e32dcc438cd 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -122,7 +122,7 @@ public: std::string & sample_path, const ContextPtr & context); - void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override; protected: String getPathSample(StorageInMemoryMetadata metadata, ContextPtr context); diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 91263180b5c..550ffa245e3 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -2114,7 +2114,7 @@ void StorageFile::truncate( } } -void StorageFile::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +void StorageFile::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const { if (checkAndGetLiteralArgument(evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context), "format") == "auto") args[0] = std::make_shared(format_name); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 5b95d1a9f8b..477d75a77a8 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -139,7 +139,7 @@ public: bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } - void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override; protected: friend class StorageFileSource; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index af91768f79e..5607054a149 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1572,7 +1572,7 @@ void StorageURL::processNamedCollectionResult(Configuration & configuration, con configuration.structure = collection.getOrDefault("structure", "auto"); } -void StorageURL::updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const +void StorageURL::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const { TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(args, "", format_name, context, /*with_structure=*/false); } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 4f3e920afac..7df5b90653d 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -305,7 +305,7 @@ public: bool supportsDynamicSubcolumns() const override { return true; } - void updateEngineArgsForCreateQuery(ASTs & args, const ContextPtr & context) const override; + void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override; static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); From c185b5bc3813ad22d45ac29e6a1adc8b71b07265 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 19 Nov 2024 22:57:53 +0000 Subject: [PATCH 03/38] Fix style --- tests/integration/test_storage_hdfs/test.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 9f0afc57ad9..d37b0d7218f 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -1225,28 +1225,31 @@ def test_format_detection(started_cluster): node.query( f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1')" ) - result = node.query( - f"show create table test_format_detection" + result = node.query(f"show create table test_format_detection") + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" ) - assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" node.query("drop table test_format_detection") node.query( f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto)" ) - result = node.query( - f"show create table test_format_detection" + result = node.query(f"show create table test_format_detection") + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" ) - assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\')\n" node.query("drop table test_format_detection") node.query( f"create table test_format_detection engine=HDFS('hdfs://hdfs1:9000/{dir}/test_format_detection1', auto, 'none')" ) - result = node.query( - f"show create table test_format_detection" + result = node.query(f"show create table test_format_detection") + assert ( + result + == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\', \\'none\\')\n" ) - assert result == f"CREATE TABLE default.test_format_detection\\n(\\n `x` Nullable(String),\\n `y` Nullable(String)\\n)\\nENGINE = HDFS(\\'hdfs://hdfs1:9000/{dir}/test_format_detection1\\', \\'JSON\\', \\'none\\')\n" def test_write_to_globbed_partitioned_path(started_cluster): From 130a1c52a252037120fdffa4d6f5b30657dba890 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:11:51 +0100 Subject: [PATCH 04/38] Update 03273_format_inference_create_query_s3_url.sql --- .../0_stateless/03273_format_inference_create_query_s3_url.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql index 353b78b1e15..5d653843ec8 100644 --- a/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql +++ b/tests/queries/0_stateless/03273_format_inference_create_query_s3_url.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + drop table if exists test; create table test engine=S3('http://localhost:11111/test/json_data'); From 28534272c9577085d2d415c664e831af3885df98 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Nov 2024 15:30:34 +0000 Subject: [PATCH 05/38] Bring back optimization for reading subcolumns of single column in Compact parts --- .../Serializations/SerializationObject.cpp | 2 + .../MergeTree/MergeTreeReaderCompact.cpp | 37 +++++++++++++------ .../MergeTree/MergeTreeReaderCompact.h | 3 +- .../MergeTreeReaderCompactSingleBuffer.cpp | 8 +++- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 1b95fddee9f..805a11521b3 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -9,6 +9,8 @@ #include #include +#include + namespace DB { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 7451374070c..06635c39838 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -148,7 +148,8 @@ void MergeTreeReaderCompact::readData( ColumnPtr & column, size_t rows_to_read, const InputStreamGetter & getter, - ISerialization::SubstreamsCache & cache) + ISerialization::SubstreamsCache & cache, + std::unordered_map & columns_cache_for_subcolumns) { try { @@ -171,17 +172,31 @@ void MergeTreeReaderCompact::readData( const auto & type_in_storage = name_and_type.getTypeInStorage(); const auto & name_in_storage = name_and_type.getNameInStorage(); - auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); - ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); - auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); - - /// TODO: Avoid extra copying. - if (column->empty()) - column = subcolumn; + if (auto cache_for_subcolumns_it = columns_cache_for_subcolumns.find(name_in_storage); cache_for_subcolumns_it != columns_cache_for_subcolumns.end()) + { + auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), cache_for_subcolumns_it->second); + /// TODO: Avoid extra copying. + if (column->empty()) + column = IColumn::mutate(subcolumn); + else + column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); + } else - column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); + { + auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); + ColumnPtr temp_column = type_in_storage->createColumn(*serialization); + + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); + auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); + + /// TODO: Avoid extra copying. + if (column->empty()) + column = subcolumn; + else + column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); + + columns_cache_for_subcolumns[name_in_storage] = temp_column; + } } else { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index 1c6bd1474e3..f18ee8808ec 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -45,7 +45,8 @@ protected: ColumnPtr & column, size_t rows_to_read, const InputStreamGetter & getter, - ISerialization::SubstreamsCache & cache); + ISerialization::SubstreamsCache & cache, + std::unordered_map & columns_cache_for_subcolumns); void readPrefix( const NameAndTypePair & name_and_type, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 649bcce1188..50224eba82d 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -29,6 +29,12 @@ try /// Use cache to avoid reading the column with the same name twice. /// It may happen if there are empty array Nested in the part. ISerialization::SubstreamsCache cache; + /// If we need to read multiple subcolumns from a single column in storage, + /// we will read it this column only once and then reuse to extract all subcolumns. + /// We cannot use SubstreamsCache for it, because we may also read the full column itself + /// and it might me not empty inside res_columns (and SubstreamsCache contains the whole columns). + /// TODO: refactor the code in a way when we first read all full columns and then extract all subcolumns from them. + std::unordered_map columns_cache_for_subcolumns; for (size_t pos = 0; pos < num_columns; ++pos) { @@ -56,7 +62,7 @@ try }; readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); - readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache, columns_cache_for_subcolumns); } ++from_mark; From e1d0932b98d32ec70a80710c3aff980c01ff4460 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Nov 2024 15:32:09 +0000 Subject: [PATCH 06/38] Remove extra include --- src/DataTypes/Serializations/SerializationObject.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 805a11521b3..1b95fddee9f 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -9,8 +9,6 @@ #include #include -#include - namespace DB { From cb833a009aa1ff5c9aa3296d76dd22a06d39e7e5 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Nov 2024 20:29:02 +0000 Subject: [PATCH 07/38] Fix tests --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 9 ++++++--- src/Storages/MergeTree/MergeTreeReaderCompact.h | 3 ++- .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 06635c39838..a4fa3a30d23 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -149,7 +149,8 @@ void MergeTreeReaderCompact::readData( size_t rows_to_read, const InputStreamGetter & getter, ISerialization::SubstreamsCache & cache, - std::unordered_map & columns_cache_for_subcolumns) + std::unordered_map & columns_cache_for_subcolumns, + const ColumnNameLevel & name_level_for_offsets) { try { @@ -172,7 +173,8 @@ void MergeTreeReaderCompact::readData( const auto & type_in_storage = name_and_type.getTypeInStorage(); const auto & name_in_storage = name_and_type.getNameInStorage(); - if (auto cache_for_subcolumns_it = columns_cache_for_subcolumns.find(name_in_storage); cache_for_subcolumns_it != columns_cache_for_subcolumns.end()) + auto cache_for_subcolumns_it = columns_cache_for_subcolumns.find(name_in_storage); + if (!name_level_for_offsets.has_value() && cache_for_subcolumns_it != columns_cache_for_subcolumns.end()) { auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), cache_for_subcolumns_it->second); /// TODO: Avoid extra copying. @@ -195,7 +197,8 @@ void MergeTreeReaderCompact::readData( else column->assumeMutable()->insertRangeFrom(*subcolumn, 0, subcolumn->size()); - columns_cache_for_subcolumns[name_in_storage] = temp_column; + if (!name_level_for_offsets.has_value()) + columns_cache_for_subcolumns[name_in_storage] = temp_column; } } else diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h index f18ee8808ec..7dbebb75a91 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -46,7 +46,8 @@ protected: size_t rows_to_read, const InputStreamGetter & getter, ISerialization::SubstreamsCache & cache, - std::unordered_map & columns_cache_for_subcolumns); + std::unordered_map & columns_cache_for_subcolumns, + const ColumnNameLevel & name_level_for_offsets); void readPrefix( const NameAndTypePair & name_and_type, diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 50224eba82d..0c3e67d5078 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -62,7 +62,7 @@ try }; readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]); - readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache, columns_cache_for_subcolumns); + readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache, columns_cache_for_subcolumns, columns_for_offsets[pos]); } ++from_mark; From 09d4d501dc49ffdaa51f17d319381fdc5ce3d5b0 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 25 Nov 2024 19:08:08 +0000 Subject: [PATCH 08/38] Add test --- tests/performance/compact_part_subcolumns.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/performance/compact_part_subcolumns.xml diff --git a/tests/performance/compact_part_subcolumns.xml b/tests/performance/compact_part_subcolumns.xml new file mode 100644 index 00000000000..234b33ab8a7 --- /dev/null +++ b/tests/performance/compact_part_subcolumns.xml @@ -0,0 +1,13 @@ + + + 1 + + + + + CREATE TABLE t_json (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000 + INSERT INTO t_json SELECT toJSONString(map(number % 10, repeat('a', number % 100))) FROM numbers(10000000) + SELECT data.k0, data.k1, data.k2, data.k3, data.k4, data.k5, data.k6, data.k7, data.k8, data.k9 FROM t_json FORMAT Null + DROP TABLE IF EXISTS t_json + From a1650e053e530a02038127c4e82522176702d881 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 26 Nov 2024 17:08:19 +0000 Subject: [PATCH 09/38] Try to fix build --- src/TableFunctions/TableFunctionURL.cpp | 39 ----------------------- src/TableFunctions/TableFunctionURL.h | 42 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 5de3a302772..cad5cac3e31 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -77,45 +77,6 @@ void TableFunctionURL::parseArgumentsImpl(ASTs & args, const ContextPtr & contex } } -void TableFunctionURL::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure) -{ - if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) - { - /// In case of named collection, just add key-value pairs "format='...', structure='...'" - /// at the end of arguments to override existed format and structure with "auto" values. - if (collection->getOrDefault("format", "auto") == "auto") - { - ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; - auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); - args.push_back(format_equal_func); - } - if (with_structure && collection->getOrDefault("structure", "auto") == "auto") - { - ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); - args.push_back(structure_equal_func); - } - } - else - { - /// If arguments contain headers, just remove it and add to the end of arguments later. - HTTPHeaderEntries tmp_headers; - size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context); - ASTPtr headers_ast; - if (count != args.size()) - { - chassert(count + 1 == args.size()); - headers_ast = args.back(); - args.pop_back(); - } - - ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, with_structure); - - if (headers_ast) - args.push_back(headers_ast); - } -} - StoragePtr TableFunctionURL::getStorage( const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context, const std::string & table_name, const String & compression_method_) const diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index d2c9d4d9ddf..d76fb9abea6 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -34,7 +37,44 @@ public: ColumnsDescription getActualTableStructure(ContextPtr context, bool is_insert_query) const override; - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context, bool with_structure) + { + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) + { + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (with_structure && collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } + } + else + { + /// If arguments contain headers, just remove it and add to the end of arguments later. + HTTPHeaderEntries tmp_headers; + size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context); + ASTPtr headers_ast; + if (count != args.size()) + { + chassert(count + 1 == args.size()); + headers_ast = args.back(); + args.pop_back(); + } + + ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(args, structure_, format_, context, with_structure); + + if (headers_ast) + args.push_back(headers_ast); + } + } protected: void parseArguments(const ASTPtr & ast, ContextPtr context) override; From 5c4da392593876391e1e6a4e7b60b9f0f61c911e Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 30 Nov 2024 22:36:19 +0100 Subject: [PATCH 10/38] Add retries while creating a replicated table. --- src/Backups/RestorerFromBackup.cpp | 13 + src/Backups/RestorerFromBackup.h | 3 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 36 +-- src/Interpreters/InterpreterSystemQuery.cpp | 11 +- .../ReplicatedMergeTreeAttachThread.cpp | 11 +- .../MergeTree/registerStorageMergeTree.cpp | 15 +- src/Storages/StorageReplicatedMergeTree.cpp | 244 ++++++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 40 ++- 9 files changed, 291 insertions(+), 84 deletions(-) diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index f907d80a64a..9b3b2408706 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,9 @@ namespace DB { namespace Setting { + extern const SettingsUInt64 backup_restore_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_retry_max_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_max_retries; extern const SettingsSeconds lock_acquire_timeout; } @@ -102,6 +106,10 @@ RestorerFromBackup::RestorerFromBackup( , after_task_callback(after_task_callback_) , create_table_timeout(context->getConfigRef().getUInt64("backups.create_table_timeout", 300000)) , log(getLogger("RestorerFromBackup")) + , zookeeper_retries_info( + context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], + context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) , tables_dependencies("RestorerFromBackup") , thread_pool(thread_pool_) { @@ -976,6 +984,11 @@ void RestorerFromBackup::createTable(const QualifiedTableName & table_name) query_context->setSetting("database_replicated_allow_explicit_uuid", 3); query_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); + /// Creating of replicated tables may need retries. + query_context->setSetting("keeper_max_retries", zookeeper_retries_info.max_retries); + query_context->setSetting("keeper_initial_backoff_ms", zookeeper_retries_info.initial_backoff_ms); + query_context->setSetting("keeper_max_backoff_ms", zookeeper_retries_info.max_backoff_ms); + /// Execute CREATE TABLE query (we call IDatabase::createTableRestoredFromBackup() to allow the database to do some /// database-specific things). database->createTableRestoredFromBackup( diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index aa5288643a0..3fb314e5c42 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -83,6 +84,7 @@ private: std::chrono::milliseconds create_table_timeout; LoggerPtr log; + const ZooKeeperRetriesInfo zookeeper_retries_info; Mode mode = Mode::RESTORE; Strings all_hosts; DDLRenamingMap renaming_map; @@ -170,7 +172,6 @@ private: TablesDependencyGraph tables_dependencies TSA_GUARDED_BY(mutex); std::vector data_restore_tasks TSA_GUARDED_BY(mutex); std::unique_ptr access_restorer TSA_GUARDED_BY(mutex); - bool access_restored TSA_GUARDED_BY(mutex) = false; std::vector> futures TSA_GUARDED_BY(mutex); std::atomic exception_caught = false; diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 3dbfd5f222d..eed29c0d821 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -408,7 +408,7 @@ void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr tab } else { - rmt->restoreMetadataInZooKeeper(); + rmt->restoreMetadataInZooKeeper(/* query_status = */ nullptr, /* zookeeper_retries_info = */ {}); LOG_INFO ( log, diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 9e81871579d..e8add48a4c2 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1633,29 +1633,29 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (isReplicated(*inner_table_engine)) is_storage_replicated = true; } - } + } - bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; - if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; + if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) + { + const bool allow_create_select_for_replicated + = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; + if (!allow_create_select_for_replicated) { - const bool allow_create_select_for_replicated - = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated; - if (!allow_create_select_for_replicated) - { - /// POPULATE can be enabled with setting, provide hint in error message - if (create.is_populate) - throw Exception( - ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT " - "queries. " - "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with " - "caution"); - + /// POPULATE can be enabled with setting, provide hint in error message + if (create.is_populate) throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, - "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); - } + "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT " + "queries. " + "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with " + "caution"); + + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries."); } + } if (create.is_clone_as) { diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9aec16a3fb7..ca96ee3245f 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -98,6 +98,9 @@ namespace DB { namespace Setting { + extern const SettingsUInt64 keeper_max_retries; + extern const SettingsUInt64 keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 keeper_retry_max_backoff_ms; extern const SettingsSeconds lock_acquire_timeout; extern const SettingsSeconds receive_timeout; extern const SettingsMaxThreads max_threads; @@ -878,7 +881,13 @@ void InterpreterSystemQuery::restoreReplica() if (table_replicated_ptr == nullptr) throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs()); - table_replicated_ptr->restoreMetadataInZooKeeper(); + const auto & settings = getContext()->getSettingsRef(); + + table_replicated_ptr->restoreMetadataInZooKeeper( + getContext()->getProcessListElementSafe(), + ZooKeeperRetriesInfo{settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms]}); } StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index c258048354e..c654b459c24 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -166,23 +166,24 @@ void ReplicatedMergeTreeAttachThread::runImpl() /// Just in case it was not removed earlier due to connection loss zookeeper->tryRemove(replica_path + "/flags/force_restore_data"); - storage.checkTableStructure(replica_path, metadata_snapshot); + /// Here `zookeeper_retries_info = {}` because the attach thread has its own retries (see ReplicatedMergeTreeAttachThread::run()). + storage.checkTableStructure(replica_path, metadata_snapshot, /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); storage.checkParts(skip_sanity_checks); /// Temporary directories contain uninitialized results of Merges or Fetches (after forced restart), /// don't allow to reinitialize them, delete each of them immediately. storage.clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); - storage.createNewZooKeeperNodes(); - storage.syncPinnedPartUUIDs(); + storage.createNewZooKeeperNodes(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); std::lock_guard lock(storage.table_shared_id_mutex); - storage.createTableSharedID(); + storage.createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); }; void ReplicatedMergeTreeAttachThread::finalizeInitialization() TSA_NO_THREAD_SAFETY_ANALYSIS { - storage.startupImpl(/* from_attach_thread */ true); + storage.startupImpl(/* from_attach_thread */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); storage.initialization_done = true; LOG_INFO(log, "Table is initialized"); } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 9f66a079998..12b5d115903 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,9 @@ namespace Setting extern const SettingsBool allow_suspicious_ttl_expressions; extern const SettingsBool create_table_empty_primary_key_by_default; extern const SettingsUInt64 database_replicated_allow_replicated_engine_arguments; + extern const SettingsUInt64 keeper_max_retries; + extern const SettingsUInt64 keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 keeper_retry_max_backoff_ms; } namespace MergeTreeSetting @@ -831,6 +835,12 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (auto txn = args.getLocalContext()->getZooKeeperMetadataTransaction()) need_check_table_structure = txn->isInitialQuery(); + ZooKeeperRetriesInfo create_query_zk_retries_info; + create_query_zk_retries_info.max_retries = local_settings[Setting::keeper_max_retries]; + create_query_zk_retries_info.initial_backoff_ms = local_settings[Setting::keeper_retry_initial_backoff_ms]; + create_query_zk_retries_info.max_backoff_ms = local_settings[Setting::keeper_retry_max_backoff_ms]; + auto create_query_status = args.getLocalContext()->getProcessListElementSafe(); + return std::make_shared( zookeeper_info, args.mode, @@ -841,8 +851,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) date_column_name, merging_params, std::move(storage_settings), - need_check_table_structure); + need_check_table_structure, + create_query_zk_retries_info, + create_query_status); } + return std::make_shared( args.table_id, args.relative_data_path, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bd476625081..ac1b37ecbdb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -366,7 +366,9 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, - bool need_check_structure) + bool need_check_structure, + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, + QueryStatusPtr create_query_status_) : MergeTreeData(table_id_, metadata_, context_, @@ -380,6 +382,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , zookeeper_path(zookeeper_info.path) , replica_name(zookeeper_info.replica_name) , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name) + , create_query_zookeeper_retries_info(create_query_zookeeper_retries_info_) + , create_query_status(create_query_status_) , reader(*this) , writer(*this) , merger_mutator(*this) @@ -569,7 +573,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( try { - bool is_first_replica = createTableIfNotExists(metadata_snapshot); + bool is_first_replica = createTableIfNotExists(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); try { @@ -578,24 +582,22 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( /// We have to check granularity on other replicas. If it's fixed we /// must create our new replica with fixed granularity and store this /// information in /replica/metadata. - other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(); + other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); /// Allow structure mismatch for secondary queries from Replicated database. /// It may happen if the table was altered just after creation. /// Metadata will be updated in cloneMetadataIfNeeded(...), metadata_version will be 0 for a while. - bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, need_check_structure); + int32_t metadata_version; + bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, &metadata_version, need_check_structure, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); if (same_structure) { - Coordination::Stat metadata_stat; - current_zookeeper->get(fs::path(zookeeper_path) / "metadata", &metadata_stat); - /** We change metadata_snapshot so that `createReplica` method will create `metadata_version` node in ZooKeeper * with version of table '/metadata' node in Zookeeper. * * Otherwise `metadata_version` for not first replica will be initialized with 0 by default. */ - setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_stat.version)); + setInMemoryMetadata(metadata_snapshot->withMetadataVersion(metadata_version)); metadata_snapshot = getInMemoryMetadataPtr(); } } @@ -607,15 +609,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } if (!is_first_replica) - createReplica(metadata_snapshot); + createReplica(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); - createNewZooKeeperNodes(); - syncPinnedPartUUIDs(); + createNewZooKeeperNodes(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + syncPinnedPartUUIDs(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); if (!has_metadata_in_zookeeper.has_value() || *has_metadata_in_zookeeper) - createTableSharedID(); - - + createTableSharedID(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); } catch (...) { @@ -628,12 +628,29 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } -bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper() +bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const { - auto zookeeper = getZooKeeper(); - String metadata_str = zookeeper->get(zookeeper_path + "/metadata"); - auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); - return metadata_from_zk.index_granularity_bytes == 0; + bool fixed_granularity = false; + + auto check_fixed_granularity = [&] + { + auto zookeeper = getZooKeeper(); + String metadata_str = zookeeper->get(zookeeper_path + "/metadata"); + auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); + fixed_granularity = (metadata_from_zk.index_granularity_bytes == 0); + }; + + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkFixedGranularityInZookeeper", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { check_fixed_granularity(); }); + } + else + { + check_fixed_granularity(); + } + + return fixed_granularity; } @@ -808,7 +825,20 @@ std::vector getAncestors(const String & path) } -void StorageReplicatedMergeTree::createNewZooKeeperNodes() +void StorageReplicatedMergeTree::createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +{ + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createNewZooKeeperNodes", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { createNewZooKeeperNodesAttempt(); }); + } + else + { + createNewZooKeeperNodesAttempt(); + } +} + +void StorageReplicatedMergeTree::createNewZooKeeperNodesAttempt() const { auto zookeeper = getZooKeeper(); @@ -873,14 +903,34 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() } } +bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, + const ZooKeeperRetriesInfo & zookeeper_retries_info, + QueryStatusPtr process_list_element) const +{ + bool table_created = false; + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableIfNotExists", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); }); + } + else + { + table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); + } + return table_created; +} -bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot) +bool StorageReplicatedMergeTree::createTableIfNotExistsAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const { auto zookeeper = getZooKeeper(); zookeeper->createAncestors(zookeeper_path); for (size_t i = 0; i < 1000; ++i) { + /// Check if the query was cancelled. + if (process_list_element) + process_list_element->checkTimeLimit(); + /// Invariant: "replicas" does not exist if there is no table or if there are leftovers from incompletely dropped table. if (zookeeper->exists(zookeeper_path + "/replicas")) { @@ -1019,7 +1069,22 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr "of wrong zookeeper_path or because of logical error"); } -void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot) +void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot, + const ZooKeeperRetriesInfo & zookeeper_retries_info, + QueryStatusPtr process_list_element) const +{ + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createReplica", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { createReplicaAttempt(metadata_snapshot, process_list_element); }); + } + else + { + createReplicaAttempt(metadata_snapshot, process_list_element); + } +} + +void StorageReplicatedMergeTree::createReplicaAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const { auto zookeeper = getZooKeeper(); @@ -1103,6 +1168,10 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada do { + /// Check if the query was cancelled. + if (process_list_element) + process_list_element->checkTimeLimit(); + Coordination::Stat replicas_stat; String replicas_value; @@ -1169,6 +1238,25 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada } while (code == Coordination::Error::ZBADVERSION); } +ZooKeeperRetriesInfo StorageReplicatedMergeTree::getCreateQueryZooKeeperRetriesInfo() const +{ + std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; + return create_query_zookeeper_retries_info; +} + +QueryStatusPtr StorageReplicatedMergeTree::getCreateQueryStatus() const +{ + std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; + return create_query_status; +} + +void StorageReplicatedMergeTree::clearCreateQueryZooKeeperRetriesInfo() +{ + std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; + create_query_zookeeper_retries_info = {}; + create_query_status = {}; +} + zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeperIfTableShutDown() const { @@ -1530,7 +1618,26 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper /** Verify that list of columns and table storage_settings_ptr match those specified in ZK (/metadata). * If not, throw an exception. */ -bool StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, bool strict_check) +bool StorageReplicatedMergeTree::checkTableStructure( + const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, + const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +{ + bool same_structure = false; + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkTableStructure", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { same_structure = checkTableStructureAttempt(zookeeper_prefix, metadata_snapshot, metadata_version, strict_check); }); + } + else + { + same_structure = checkTableStructureAttempt(zookeeper_prefix, metadata_snapshot, metadata_version, strict_check); + } + return same_structure; +} + + +bool StorageReplicatedMergeTree::checkTableStructureAttempt( + const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check) const { auto zookeeper = getZooKeeper(); @@ -1541,6 +1648,9 @@ bool StorageReplicatedMergeTree::checkTableStructure(const String & zookeeper_pr auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); old_metadata.checkEquals(metadata_from_zk, metadata_snapshot->getColumns(), getContext()); + if (metadata_version) + *metadata_version = metadata_stat.version; + Coordination::Stat columns_stat; auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(fs::path(zookeeper_prefix) / "columns", &columns_stat)); @@ -1860,21 +1970,35 @@ bool StorageReplicatedMergeTree::checkPartsImpl(bool skip_sanity_checks) } -void StorageReplicatedMergeTree::syncPinnedPartUUIDs() +void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) { - auto zookeeper = getZooKeeper(); + String new_pinned_part_uuids_str; + Coordination::Stat new_stat; - Coordination::Stat stat; - String s = zookeeper->get(zookeeper_path + "/pinned_part_uuids", &stat); + auto read_pinned_part_uuids = [&] + { + auto zookeeper = getZooKeeper(); + new_pinned_part_uuids_str = zookeeper->get(zookeeper_path + "/pinned_part_uuids", &new_stat); + }; + + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::syncPinnedPartUUIDs", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { read_pinned_part_uuids(); }); + } + else + { + read_pinned_part_uuids(); + } std::lock_guard lock(pinned_part_uuids_mutex); /// Unsure whether or not this can be called concurrently. - if (pinned_part_uuids->stat.version < stat.version) + if (pinned_part_uuids->stat.version < new_stat.version) { auto new_pinned_part_uuids = std::make_shared(); - new_pinned_part_uuids->fromString(s); - new_pinned_part_uuids->stat = stat; + new_pinned_part_uuids->fromString(new_pinned_part_uuids_str); + new_pinned_part_uuids->stat = new_stat; pinned_part_uuids = new_pinned_part_uuids; } @@ -2228,7 +2352,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) case LogEntry::ALTER_METADATA: return executeMetadataAlter(entry); case LogEntry::SYNC_PINNED_PART_UUIDS: - syncPinnedPartUUIDs(); + syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); return true; case LogEntry::CLONE_PART_FROM_SHARD: executeClonePartFromShard(entry); @@ -4377,17 +4501,29 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n } -void StorageReplicatedMergeTree::startBeingLeader() +void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) { - auto zookeeper = getZooKeeper(); - if (!(*getSettings())[MergeTreeSetting::replicated_can_become_leader]) { LOG_INFO(log, "Will not enter leader election because replicated_can_become_leader=0"); return; } - zkutil::checkNoOldLeaders(log.load(), *zookeeper, fs::path(zookeeper_path) / "leader_election"); + auto start_being_leader = [&] + { + auto zookeeper = getZooKeeper(); + zkutil::checkNoOldLeaders(log.load(), *zookeeper, fs::path(zookeeper_path) / "leader_election"); + }; + + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::startBeingLeader", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { start_being_leader(); }); + } + else + { + start_being_leader(); + } LOG_INFO(log, "Became leader"); is_leader = true; @@ -5260,10 +5396,10 @@ void StorageReplicatedMergeTree::startup() return; } - startupImpl(/* from_attach_thread */ false); + startupImpl(/* from_attach_thread */ false, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); } -void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread) +void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) { /// Do not start replication if ZooKeeper is not configured or there is no metadata in zookeeper if (!has_metadata_in_zookeeper.has_value() || !*has_metadata_in_zookeeper) @@ -5292,7 +5428,7 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread) getContext()->getInterserverIOHandler().addEndpoint( data_parts_exchange_ptr->getId(getEndpointName()), data_parts_exchange_ptr); - startBeingLeader(); + startBeingLeader(zookeeper_retries_info, process_list_element); if (from_attach_thread) { @@ -5330,6 +5466,9 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread) startBackgroundMovesIfNeeded(); part_moves_between_shards_orchestrator.start(); + + /// After finishing startup() create_query_zk_retries_info won't be used anymore. + clearCreateQueryZooKeeperRetriesInfo(); } catch (...) { @@ -6544,7 +6683,7 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition( return true; } -void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() +void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info) { LOG_INFO(log, "Restoring replica metadata"); @@ -6587,14 +6726,14 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() LOG_INFO(log, "Moved all parts to detached/"); - const bool is_first_replica = createTableIfNotExists(metadata_snapshot); + const bool is_first_replica = createTableIfNotExists(metadata_snapshot, zookeeper_retries_info, query_status); LOG_INFO(log, "Created initial ZK nodes, replica is first: {}", is_first_replica); if (!is_first_replica) - createReplica(metadata_snapshot); + createReplica(metadata_snapshot, zookeeper_retries_info, query_status); - createNewZooKeeperNodes(); + createNewZooKeeperNodes(zookeeper_retries_info, query_status); LOG_INFO(log, "Created ZK nodes for table"); @@ -6606,7 +6745,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() LOG_INFO(log, "Attached all partitions, starting table"); - startupImpl(/* from_attach_thread */ false); + startupImpl(/* from_attach_thread */ false, zookeeper_retries_info, query_status); } void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) @@ -8785,7 +8924,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( { /// Optimistic check that for compatible destination table structure. - checkTableStructure(to, getInMemoryMetadataPtr()); + checkTableStructure(to, getInMemoryMetadataPtr(), /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); } PinnedPartUUIDs src_pins; @@ -9388,7 +9527,7 @@ String StorageReplicatedMergeTree::getTableSharedID() const { /// Can happen if table was partially initialized before drop by DatabaseCatalog if (table_shared_id == UUIDHelpers::Nil) - createTableSharedID(); + createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); } else { @@ -9403,7 +9542,20 @@ std::map StorageReplicatedMergeTree::getUnfinishe return queue.getUnfinishedMutations(); } -void StorageReplicatedMergeTree::createTableSharedID() const +void StorageReplicatedMergeTree::createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +{ + if (zookeeper_retries_info.max_retries > 0) + { + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableSharedID", log.load(), zookeeper_retries_info, process_list_element}; + retries_ctl.retryLoop([&] { createTableSharedIDAttempt(); }); + } + else + { + createTableSharedIDAttempt(); + } +} + +void StorageReplicatedMergeTree::createTableSharedIDAttempt() const { LOG_DEBUG(log, "Creating shared ID for table {}", getStorageID().getNameForLogs()); // can be set by the call to getTableSharedID diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index a790e548645..c6081d45bf4 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,9 @@ public: const String & date_column_name, const MergingParams & merging_params_, std::unique_ptr settings_, - bool need_check_structure); + bool need_check_structure, + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, + QueryStatusPtr create_query_status_); void startup() override; @@ -313,7 +316,7 @@ public: /// Restores table metadata if ZooKeeper lost it. /// Used only on restarted readonly replicas (not checked). All active (Active) parts are moved to detached/ /// folder and attached. Parts in all other states are just moved to detached/ folder. - void restoreMetadataInZooKeeper(); + void restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info); /// Get throttler for replicated fetches ThrottlerPtr getFetchesThrottler() const @@ -424,6 +427,10 @@ private: const String replica_name; // shorthand for zookeeper_info.replica_name const String replica_path; + ZooKeeperRetriesInfo create_query_zookeeper_retries_info TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); + QueryStatusPtr create_query_status TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); + mutable std::mutex create_query_zookeeper_retries_info_mutex; + /** /replicas/me/is_active. */ zkutil::EphemeralNodeHolderPtr replica_is_active_node; @@ -572,18 +579,28 @@ private: /** Creates the minimum set of nodes in ZooKeeper and create first replica. * Returns true if was created, false if exists. */ - bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot); + bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool createTableIfNotExistsAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** * Creates a replica in ZooKeeper and adds to the queue all that it takes to catch up with the rest of the replicas. */ - void createReplica(const StorageMetadataPtr & metadata_snapshot); + void createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createReplicaAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** Create nodes in the ZK, which must always be, but which might not exist when older versions of the server are running. */ - void createNewZooKeeperNodes(); + void createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createNewZooKeeperNodesAttempt() const; - bool checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, bool strict_check = true); + /// Returns the ZooKeeper retries info specified for the CREATE TABLE query which is creating and starting this table right now. + ZooKeeperRetriesInfo getCreateQueryZooKeeperRetriesInfo() const; + QueryStatusPtr getCreateQueryStatus() const; + void clearCreateQueryZooKeeperRetriesInfo(); + + bool checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, + const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool checkTableStructureAttempt(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check) const; /// A part of ALTER: apply metadata changes only (data parts are altered separately). /// Must be called under IStorage::lockForAlter() lock. @@ -602,7 +619,7 @@ private: /// Synchronize the list of part uuids which are currently pinned. These should be sent to root query executor /// to be used for deduplication. - void syncPinnedPartUUIDs(); + void syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); /** Check that the part's checksum is the same as the checksum of the same part on some other replica. * If no one has such a part, nothing checks. @@ -705,7 +722,7 @@ private: /// Start being leader (if not disabled by setting). /// Since multi-leaders are allowed, it just sets is_leader flag. - void startBeingLeader(); + void startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); void stopBeingLeader(); /** Selects the parts to merge and writes to the log. @@ -922,7 +939,7 @@ private: /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed - bool checkFixedGranularityInZookeeper(); + bool checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; /// Wait for timeout seconds mutation is finished on replicas void waitMutationToFinishOnReplicas( @@ -960,7 +977,8 @@ private: void createAndStoreFreezeMetadata(DiskPtr disk, DataPartPtr part, String backup_part_path) const override; // Create table id if needed - void createTableSharedID() const; + void createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createTableSharedIDAttempt() const; bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica); void watchZeroCopyLock(const String & part_name, const DiskPtr & disk); @@ -976,7 +994,7 @@ private: /// Or if node actually disappeared. bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override; - void startupImpl(bool from_attach_thread); + void startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); std::vector getZookeeperZeroCopyLockPaths() const; static void dropZookeeperZeroCopyLockPaths(zkutil::ZooKeeperPtr zookeeper, From 136c5de31c79a41a9a2f006b00b94ce6de8af976 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 1 Dec 2024 23:21:10 +0100 Subject: [PATCH 11/38] Add field query_status to the ZooKeeperRetriesInfo structure. --- src/Backups/BackupCoordinationOnCluster.cpp | 4 +- src/Backups/BackupCoordinationOnCluster.h | 3 +- src/Backups/BackupEntriesCollector.cpp | 8 +- src/Backups/BackupEntriesCollector.h | 4 +- src/Backups/RestoreCoordinationOnCluster.cpp | 4 +- src/Backups/RestoreCoordinationOnCluster.h | 3 +- src/Backups/RestorerFromBackup.cpp | 3 +- src/Backups/WithRetries.cpp | 5 +- src/Common/ZooKeeper/ZooKeeperRetries.h | 24 ++--- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Databases/DatabaseReplicatedWorker.cpp | 2 +- src/Databases/DatabaseReplicatedWorker.h | 2 +- src/Interpreters/DDLWorker.cpp | 4 +- src/Interpreters/DDLWorker.h | 2 +- .../DistributedQueryStatusSource.cpp | 11 +-- .../DistributedQueryStatusSource.h | 2 +- src/Interpreters/InterpreterSystemQuery.cpp | 4 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 2 +- .../ReplicatedMergeTreeAttachThread.cpp | 10 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 8 +- .../MergeTree/registerStorageMergeTree.cpp | 13 ++- src/Storages/StorageKeeperMap.cpp | 40 +++++--- src/Storages/StorageReplicatedMergeTree.cpp | 99 ++++++++----------- src/Storages/StorageReplicatedMergeTree.h | 25 +++-- .../System/StorageSystemZooKeeper.cpp | 7 +- 25 files changed, 148 insertions(+), 143 deletions(-) diff --git a/src/Backups/BackupCoordinationOnCluster.cpp b/src/Backups/BackupCoordinationOnCluster.cpp index 1b14f226eff..4fee3bb952e 100644 --- a/src/Backups/BackupCoordinationOnCluster.cpp +++ b/src/Backups/BackupCoordinationOnCluster.cpp @@ -182,6 +182,7 @@ BackupCoordinationOnCluster::BackupCoordinationOnCluster( , current_host(current_host_) , current_host_index(findCurrentHostIndex(current_host, all_hosts)) , plain_backup(is_plain_backup_) + , process_list_element(process_list_element_) , log(getLogger("BackupCoordinationOnCluster")) , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); }) , cleaner(/* is_restore = */ false, zookeeper_path, with_retries, log) @@ -273,7 +274,8 @@ ZooKeeperRetriesInfo BackupCoordinationOnCluster::getOnClusterInitializationKeep { return ZooKeeperRetriesInfo{keeper_settings.max_retries_while_initializing, static_cast(keeper_settings.retry_initial_backoff_ms.count()), - static_cast(keeper_settings.retry_max_backoff_ms.count())}; + static_cast(keeper_settings.retry_max_backoff_ms.count()), + process_list_element}; } void BackupCoordinationOnCluster::serializeToMultipleZooKeeperNodes(const String & path, const String & value, const String & logging_name) diff --git a/src/Backups/BackupCoordinationOnCluster.h b/src/Backups/BackupCoordinationOnCluster.h index b439ab619d8..9bfc33d9fbf 100644 --- a/src/Backups/BackupCoordinationOnCluster.h +++ b/src/Backups/BackupCoordinationOnCluster.h @@ -107,7 +107,8 @@ private: const String current_host; const size_t current_host_index; const bool plain_backup; - LoggerPtr const log; + const QueryStatusPtr process_list_element; + const LoggerPtr log; /// The order is important: `stage_sync` must be initialized after `with_retries` and `cleaner`. const WithRetries with_retries; diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 00a4471d994..1ff190c171d 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -111,10 +111,11 @@ BackupEntriesCollector::BackupEntriesCollector( context->getConfigRef().getUInt64("backups.max_sleep_before_next_attempt_to_collect_metadata", 5000)) , compare_collected_metadata(context->getConfigRef().getBool("backups.compare_collected_metadata", true)) , log(getLogger("BackupEntriesCollector")) - , global_zookeeper_retries_info( + , zookeeper_retries_info( context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], - context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms], + context->getProcessListElementSafe()) , threadpool(threadpool_) { } @@ -582,8 +583,7 @@ std::vector> BackupEntriesCollector::findTablesInD try { /// Database or table could be replicated - so may use ZooKeeper. We need to retry. - auto zookeeper_retries_info = global_zookeeper_retries_info; - ZooKeeperRetriesControl retries_ctl("getTablesForBackup", log, zookeeper_retries_info, nullptr); + ZooKeeperRetriesControl retries_ctl("getTablesForBackup", log, zookeeper_retries_info); retries_ctl.retryLoop([&](){ db_tables = database->getTablesForBackup(filter_by_table_name, context); }); } catch (Exception & e) diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 504489cce6b..d8b2dfd38e7 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -48,7 +48,7 @@ public: std::shared_ptr getBackupCoordination() const { return backup_coordination; } const ReadSettings & getReadSettings() const { return read_settings; } ContextPtr getContext() const { return context; } - const ZooKeeperRetriesInfo & getZooKeeperRetriesInfo() const { return global_zookeeper_retries_info; } + const ZooKeeperRetriesInfo & getZooKeeperRetriesInfo() const { return zookeeper_retries_info; } /// Returns all access entities which can be put into a backup. std::unordered_map getAllAccessEntities(); @@ -129,7 +129,7 @@ private: LoggerPtr log; /// Unfortunately we can use ZooKeeper for collecting information for backup /// and we need to retry... - ZooKeeperRetriesInfo global_zookeeper_retries_info; + ZooKeeperRetriesInfo zookeeper_retries_info; Strings all_hosts; DDLRenamingMap renaming_map; diff --git a/src/Backups/RestoreCoordinationOnCluster.cpp b/src/Backups/RestoreCoordinationOnCluster.cpp index fad7341c044..c85cca45fa7 100644 --- a/src/Backups/RestoreCoordinationOnCluster.cpp +++ b/src/Backups/RestoreCoordinationOnCluster.cpp @@ -33,6 +33,7 @@ RestoreCoordinationOnCluster::RestoreCoordinationOnCluster( , all_hosts_without_initiator(BackupCoordinationOnCluster::excludeInitiator(all_hosts)) , current_host(current_host_) , current_host_index(BackupCoordinationOnCluster::findCurrentHostIndex(current_host, all_hosts)) + , process_list_element(process_list_element_) , log(getLogger("RestoreCoordinationOnCluster")) , with_retries(log, get_zookeeper_, keeper_settings, process_list_element_, [root_zookeeper_path_](Coordination::ZooKeeperWithFaultInjection::Ptr zk) { zk->sync(root_zookeeper_path_); }) , cleaner(/* is_restore = */ true, zookeeper_path, with_retries, log) @@ -122,7 +123,8 @@ ZooKeeperRetriesInfo RestoreCoordinationOnCluster::getOnClusterInitializationKee { return ZooKeeperRetriesInfo{keeper_settings.max_retries_while_initializing, static_cast(keeper_settings.retry_initial_backoff_ms.count()), - static_cast(keeper_settings.retry_max_backoff_ms.count())}; + static_cast(keeper_settings.retry_max_backoff_ms.count()), + process_list_element}; } bool RestoreCoordinationOnCluster::acquireCreatingTableInReplicatedDatabase(const String & database_zk_path, const String & table_name) diff --git a/src/Backups/RestoreCoordinationOnCluster.h b/src/Backups/RestoreCoordinationOnCluster.h index 99929cbdac3..890117e63e8 100644 --- a/src/Backups/RestoreCoordinationOnCluster.h +++ b/src/Backups/RestoreCoordinationOnCluster.h @@ -75,7 +75,8 @@ private: const Strings all_hosts_without_initiator; const String current_host; const size_t current_host_index; - LoggerPtr const log; + const QueryStatusPtr process_list_element; + const LoggerPtr log; /// The order is important: `stage_sync` must be initialized after `with_retries` and `cleaner`. const WithRetries with_retries; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 9b3b2408706..249e4353296 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -109,7 +109,8 @@ RestorerFromBackup::RestorerFromBackup( , zookeeper_retries_info( context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], - context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms], + context->getProcessListElementSafe()) , tables_dependencies("RestorerFromBackup") , thread_pool(thread_pool_) { diff --git a/src/Backups/WithRetries.cpp b/src/Backups/WithRetries.cpp index 9c18be3ca9e..53613b2149e 100644 --- a/src/Backups/WithRetries.cpp +++ b/src/Backups/WithRetries.cpp @@ -20,9 +20,10 @@ WithRetries::RetriesControlHolder::RetriesControlHolder(const WithRetries * pare : (kind == kErrorHandling) ? parent->settings.max_retries_while_handling_error : parent->settings.max_retries, parent->settings.retry_initial_backoff_ms.count(), - parent->settings.retry_max_backoff_ms.count()) + parent->settings.retry_max_backoff_ms.count(), + (kind == kErrorHandling) ? nullptr : parent->process_list_element) /// We don't use process_list_element while handling an error because the error handling can't be cancellable. - , retries_ctl(name, parent->log, info, (kind == kErrorHandling) ? nullptr : parent->process_list_element) + , retries_ctl(name, parent->log, info) , faulty_zookeeper(parent->getFaultyZooKeeper()) {} diff --git a/src/Common/ZooKeeper/ZooKeeperRetries.h b/src/Common/ZooKeeper/ZooKeeperRetries.h index acea521a7ce..9482f72cba0 100644 --- a/src/Common/ZooKeeper/ZooKeeperRetries.h +++ b/src/Common/ZooKeeper/ZooKeeperRetries.h @@ -16,21 +16,25 @@ namespace ErrorCodes struct ZooKeeperRetriesInfo { ZooKeeperRetriesInfo() = default; - ZooKeeperRetriesInfo(UInt64 max_retries_, UInt64 initial_backoff_ms_, UInt64 max_backoff_ms_) + + ZooKeeperRetriesInfo(UInt64 max_retries_, UInt64 initial_backoff_ms_, UInt64 max_backoff_ms_, QueryStatusPtr query_status_) : max_retries(max_retries_), initial_backoff_ms(std::min(initial_backoff_ms_, max_backoff_ms_)), max_backoff_ms(max_backoff_ms_) + , query_status(query_status_) { } UInt64 max_retries = 0; /// "max_retries = 0" means only one attempt. - UInt64 initial_backoff_ms = 100; - UInt64 max_backoff_ms = 5000; + UInt64 initial_backoff_ms = 0; + UInt64 max_backoff_ms = 0; + + QueryStatusPtr query_status; /// can be nullptr }; class ZooKeeperRetriesControl { public: - ZooKeeperRetriesControl(std::string name_, LoggerPtr logger_, ZooKeeperRetriesInfo retries_info_, QueryStatusPtr elem) - : name(std::move(name_)), logger(logger_), retries_info(retries_info_), process_list_element(elem) + ZooKeeperRetriesControl(std::string name_, LoggerPtr logger_, ZooKeeperRetriesInfo retries_info_) + : name(std::move(name_)), logger(logger_), retries_info(retries_info_) { } @@ -39,7 +43,6 @@ public: , logger(other.logger) , retries_info(other.retries_info) , total_failures(other.total_failures) - , process_list_element(other.process_list_element) , current_backoff_ms(other.current_backoff_ms) { } @@ -222,8 +225,8 @@ private: } /// Check if the query was cancelled. - if (process_list_element) - process_list_element->checkTimeLimit(); + if (retries_info.query_status) + retries_info.query_status->checkTimeLimit(); /// retries logLastError("will retry due to error"); @@ -231,8 +234,8 @@ private: current_backoff_ms = std::min(current_backoff_ms * 2, retries_info.max_backoff_ms); /// Check if the query was cancelled again after sleeping. - if (process_list_element) - process_list_element->checkTimeLimit(); + if (retries_info.query_status) + retries_info.query_status->checkTimeLimit(); return true; } @@ -288,7 +291,6 @@ private: std::function action_after_last_failed_retry = []() {}; bool iteration_succeeded = true; bool stop_retries = false; - QueryStatusPtr process_list_element; UInt64 current_iteration = 0; UInt64 current_backoff_ms = 0; diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index eed29c0d821..6c9f5b82116 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -408,7 +408,7 @@ void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr tab } else { - rmt->restoreMetadataInZooKeeper(/* query_status = */ nullptr, /* zookeeper_retries_info = */ {}); + rmt->restoreMetadataInZooKeeper(/* zookeeper_retries_info = */ {}); LOG_INFO ( log, diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 6a711c92332..0000096c1c1 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -199,7 +199,7 @@ void DatabaseReplicatedDDLWorker::initializeReplication() active_node_holder = zkutil::EphemeralNodeHolder::existing(active_path, *active_node_holder_zookeeper); } -String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &, QueryStatusPtr) +String DatabaseReplicatedDDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &) { auto zookeeper = getAndSetZooKeeper(); return enqueueQueryImpl(zookeeper, entry, database); diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h index d2385cbdba3..2bb603354ac 100644 --- a/src/Databases/DatabaseReplicatedWorker.h +++ b/src/Databases/DatabaseReplicatedWorker.h @@ -24,7 +24,7 @@ class DatabaseReplicatedDDLWorker : public DDLWorker public: DatabaseReplicatedDDLWorker(DatabaseReplicated * db, ContextPtr context_); - String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &, QueryStatusPtr) override; + String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo &) override; String tryEnqueueAndExecuteEntry(DDLLogEntry & entry, ContextPtr query_context); diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index b1c7635b62b..3b4d31996eb 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -1054,12 +1054,12 @@ void DDLWorker::createStatusDirs(const std::string & node_path, const ZooKeeperP } -String DDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info, QueryStatusPtr process_list_element) +String DDLWorker::enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info) { String node_path; if (retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"DDLWorker::enqueueQuery", log, retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"DDLWorker::enqueueQuery", log, retries_info}; retries_ctl.retryLoop([&]{ node_path = enqueueQueryAttempt(entry); }); diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index a5f47a51bb3..ec697046d03 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -68,7 +68,7 @@ public: virtual ~DDLWorker(); /// Pushes query into DDL queue, returns path to created node - virtual String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info, QueryStatusPtr process_list_element); + virtual String enqueueQuery(DDLLogEntry & entry, const ZooKeeperRetriesInfo & retries_info); /// Host ID (name:port) for logging purposes /// Note that in each task hosts are identified individually by name:port from initiator server cluster config diff --git a/src/Interpreters/DistributedQueryStatusSource.cpp b/src/Interpreters/DistributedQueryStatusSource.cpp index 83701d41c57..aaddb1bd4e2 100644 --- a/src/Interpreters/DistributedQueryStatusSource.cpp +++ b/src/Interpreters/DistributedQueryStatusSource.cpp @@ -133,8 +133,7 @@ ExecutionStatus DistributedQueryStatusSource::getExecutionStatus(const fs::path String status_data; bool finished_exists = false; - auto retries_ctl = ZooKeeperRetriesControl( - "executeDDLQueryOnCluster", getLogger("DDLQueryStatusSource"), getRetriesInfo(), context->getProcessListElement()); + auto retries_ctl = ZooKeeperRetriesControl("executeDDLQueryOnCluster", getLogger("DDLQueryStatusSource"), getRetriesInfo()); retries_ctl.retryLoop([&]() { finished_exists = context->getZooKeeper()->tryGet(status_path, status_data); }); if (finished_exists) status.tryDeserializeText(status_data); @@ -142,13 +141,14 @@ ExecutionStatus DistributedQueryStatusSource::getExecutionStatus(const fs::path return status; } -ZooKeeperRetriesInfo DistributedQueryStatusSource::getRetriesInfo() +ZooKeeperRetriesInfo DistributedQueryStatusSource::getRetriesInfo() const { const auto & config_ref = Context::getGlobalContextInstance()->getConfigRef(); return ZooKeeperRetriesInfo( config_ref.getInt("distributed_ddl_keeper_max_retries", 5), config_ref.getInt("distributed_ddl_keeper_initial_backoff_ms", 100), - config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000)); + config_ref.getInt("distributed_ddl_keeper_max_backoff_ms", 5000), + context->getProcessListElement()); } std::pair DistributedQueryStatusSource::parseHostAndPort(const String & host_id) @@ -194,8 +194,7 @@ Chunk DistributedQueryStatusSource::generate() Strings tmp_active_hosts; { - auto retries_ctl = ZooKeeperRetriesControl( - "executeDistributedQueryOnCluster", getLogger(getName()), getRetriesInfo(), context->getProcessListElement()); + auto retries_ctl = ZooKeeperRetriesControl("executeDistributedQueryOnCluster", getLogger(getName()), getRetriesInfo()); retries_ctl.retryLoop( [&]() { diff --git a/src/Interpreters/DistributedQueryStatusSource.h b/src/Interpreters/DistributedQueryStatusSource.h index 4f58085a1f0..71315c5cd74 100644 --- a/src/Interpreters/DistributedQueryStatusSource.h +++ b/src/Interpreters/DistributedQueryStatusSource.h @@ -38,7 +38,7 @@ protected: Strings getNewAndUpdate(const Strings & current_finished_hosts); ExecutionStatus getExecutionStatus(const fs::path & status_path); - static ZooKeeperRetriesInfo getRetriesInfo(); + ZooKeeperRetriesInfo getRetriesInfo() const; static std::pair parseHostAndPort(const String & host_id); String node_path; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index ca96ee3245f..ac5f03e3b24 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -884,10 +884,10 @@ void InterpreterSystemQuery::restoreReplica() const auto & settings = getContext()->getSettingsRef(); table_replicated_ptr->restoreMetadataInZooKeeper( - getContext()->getProcessListElementSafe(), ZooKeeperRetriesInfo{settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], - settings[Setting::keeper_retry_max_backoff_ms]}); + settings[Setting::keeper_retry_max_backoff_ms], + getContext()->getProcessListElementSafe()}); } StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context) diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 0b88d07148c..43ffa946a57 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -189,7 +189,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, entry.setSettingsIfRequired(context); entry.tracing_context = OpenTelemetry::CurrentContext(); entry.initial_query_id = context->getClientInfo().initial_query_id; - String node_path = ddl_worker.enqueueQuery(entry, params.retries_info, context->getProcessListElement()); + String node_path = ddl_worker.enqueueQuery(entry, params.retries_info); return getDDLOnClusterStatus(node_path, ddl_worker.getReplicasDir(), entry, context); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp index c654b459c24..db5b1d5d0c9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp @@ -167,23 +167,23 @@ void ReplicatedMergeTreeAttachThread::runImpl() zookeeper->tryRemove(replica_path + "/flags/force_restore_data"); /// Here `zookeeper_retries_info = {}` because the attach thread has its own retries (see ReplicatedMergeTreeAttachThread::run()). - storage.checkTableStructure(replica_path, metadata_snapshot, /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.checkTableStructure(replica_path, metadata_snapshot, /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}); storage.checkParts(skip_sanity_checks); /// Temporary directories contain uninitialized results of Merges or Fetches (after forced restart), /// don't allow to reinitialize them, delete each of them immediately. storage.clearOldTemporaryDirectories(0, {"tmp_", "delete_tmp_", "tmp-fetch_"}); - storage.createNewZooKeeperNodes(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); - storage.syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.createNewZooKeeperNodes(/* zookeeper_retries_info = */ {}); + storage.syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}); std::lock_guard lock(storage.table_shared_id_mutex); - storage.createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.createTableSharedID(/* zookeeper_retries_info = */ {}); }; void ReplicatedMergeTreeAttachThread::finalizeInitialization() TSA_NO_THREAD_SAFETY_ANALYSIS { - storage.startupImpl(/* from_attach_thread */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + storage.startupImpl(/* from_attach_thread */ true, /* zookeeper_retries_info = */ {}); storage.initialization_done = true; LOG_INFO(log, "Table is initialized"); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 19a69eb46be..3422e534f7d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -201,8 +201,8 @@ size_t ReplicatedMergeTreeSinkImpl::checkQuorumPrecondition(const log, {settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]}, - context->getProcessListElement()); + settings[Setting::insert_keeper_retry_max_backoff_ms], + context->getProcessListElement()}); quorum_retries_ctl.retryLoop( [&]() { @@ -725,8 +725,8 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: log, {settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]}, - context->getProcessListElement()); + settings[Setting::insert_keeper_retry_max_backoff_ms], + context->getProcessListElement()}); auto resolve_duplicate_stage = [&] () -> CommitRetryContext::Stages { diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 12b5d115903..709f019a005 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -835,11 +835,11 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (auto txn = args.getLocalContext()->getZooKeeperMetadataTransaction()) need_check_table_structure = txn->isInitialQuery(); - ZooKeeperRetriesInfo create_query_zk_retries_info; - create_query_zk_retries_info.max_retries = local_settings[Setting::keeper_max_retries]; - create_query_zk_retries_info.initial_backoff_ms = local_settings[Setting::keeper_retry_initial_backoff_ms]; - create_query_zk_retries_info.max_backoff_ms = local_settings[Setting::keeper_retry_max_backoff_ms]; - auto create_query_status = args.getLocalContext()->getProcessListElementSafe(); + ZooKeeperRetriesInfo create_query_zk_retries_info{ + local_settings[Setting::keeper_max_retries], + local_settings[Setting::keeper_retry_initial_backoff_ms], + local_settings[Setting::keeper_retry_max_backoff_ms], + args.getLocalContext()->getProcessListElementSafe()}; return std::make_shared( zookeeper_info, @@ -852,8 +852,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) merging_params, std::move(storage_settings), need_check_table_structure, - create_query_zk_retries_info, - create_query_status); + create_query_zk_retries_info); } return std::make_shared( diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 2a4a5f3370f..1504656bce5 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -189,8 +189,8 @@ public: ZooKeeperRetriesInfo{ settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]}, - context->getProcessListElement()}; + settings[Setting::insert_keeper_retry_max_backoff_ms], + context->getProcessListElement()}}; zk_retry.retryLoop([&]() { @@ -425,8 +425,10 @@ StorageKeeperMap::StorageKeeperMap( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - context_->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + context_->getProcessListElement()}}; zk_retry.retryLoop( [&] @@ -670,8 +672,10 @@ Pipe StorageKeeperMap::read( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - context_->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + context_->getProcessListElement()}}; std::vector children; zk_retry.retryLoop([&] @@ -699,8 +703,10 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; zk_retry.retryLoop([&] { @@ -1136,8 +1142,10 @@ StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus(const ContextPtr getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; zk_retry.retryLoop([&] { @@ -1248,8 +1256,10 @@ Chunk StorageKeeperMap::getBySerializedKeys( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; zkutil::ZooKeeper::MultiTryGetResponse values; zk_retry.retryLoop([&]{ @@ -1394,8 +1404,10 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, - local_context->getProcessListElement()}; + settings[Setting::keeper_max_retries], + settings[Setting::keeper_retry_initial_backoff_ms], + settings[Setting::keeper_retry_max_backoff_ms], + local_context->getProcessListElement()}}; Coordination::Error status; zk_retry.retryLoop([&] diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ac1b37ecbdb..b57dcd144f2 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -367,8 +367,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const MergingParams & merging_params_, std::unique_ptr settings_, bool need_check_structure, - const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, - QueryStatusPtr create_query_status_) + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_) : MergeTreeData(table_id_, metadata_, context_, @@ -383,7 +382,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( , replica_name(zookeeper_info.replica_name) , replica_path(fs::path(zookeeper_path) / "replicas" / replica_name) , create_query_zookeeper_retries_info(create_query_zookeeper_retries_info_) - , create_query_status(create_query_status_) , reader(*this) , writer(*this) , merger_mutator(*this) @@ -573,7 +571,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( try { - bool is_first_replica = createTableIfNotExists(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + bool is_first_replica = createTableIfNotExists(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo()); try { @@ -582,13 +580,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( /// We have to check granularity on other replicas. If it's fixed we /// must create our new replica with fixed granularity and store this /// information in /replica/metadata. - other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + other_replicas_fixed_granularity = checkFixedGranularityInZookeeper(getCreateQueryZooKeeperRetriesInfo()); /// Allow structure mismatch for secondary queries from Replicated database. /// It may happen if the table was altered just after creation. /// Metadata will be updated in cloneMetadataIfNeeded(...), metadata_version will be 0 for a while. int32_t metadata_version; - bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, &metadata_version, need_check_structure, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + bool same_structure = checkTableStructure(zookeeper_path, metadata_snapshot, &metadata_version, need_check_structure, getCreateQueryZooKeeperRetriesInfo()); if (same_structure) { @@ -609,13 +607,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } if (!is_first_replica) - createReplica(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + createReplica(metadata_snapshot, getCreateQueryZooKeeperRetriesInfo()); - createNewZooKeeperNodes(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); - syncPinnedPartUUIDs(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + createNewZooKeeperNodes(getCreateQueryZooKeeperRetriesInfo()); + syncPinnedPartUUIDs(getCreateQueryZooKeeperRetriesInfo()); if (!has_metadata_in_zookeeper.has_value() || *has_metadata_in_zookeeper) - createTableSharedID(getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + createTableSharedID(getCreateQueryZooKeeperRetriesInfo()); } catch (...) { @@ -628,7 +626,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } -bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info) const { bool fixed_granularity = false; @@ -642,7 +640,7 @@ bool StorageReplicatedMergeTree::checkFixedGranularityInZookeeper(const ZooKeepe if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkFixedGranularityInZookeeper", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkFixedGranularityInZookeeper", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { check_fixed_granularity(); }); } else @@ -825,11 +823,11 @@ std::vector getAncestors(const String & path) } -void StorageReplicatedMergeTree::createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +void StorageReplicatedMergeTree::createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info) const { if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createNewZooKeeperNodes", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createNewZooKeeperNodes", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { createNewZooKeeperNodesAttempt(); }); } else @@ -903,19 +901,17 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodesAttempt() const } } -bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, - const ZooKeeperRetriesInfo & zookeeper_retries_info, - QueryStatusPtr process_list_element) const +bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const { bool table_created = false; if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableIfNotExists", log.load(), zookeeper_retries_info, process_list_element}; - retries_ctl.retryLoop([&] { table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); }); + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableIfNotExists", log.load(), zookeeper_retries_info}; + retries_ctl.retryLoop([&] { table_created = createTableIfNotExistsAttempt(metadata_snapshot, zookeeper_retries_info.query_status); }); } else { - table_created = createTableIfNotExistsAttempt(metadata_snapshot, process_list_element); + table_created = createTableIfNotExistsAttempt(metadata_snapshot, zookeeper_retries_info.query_status); } return table_created; } @@ -1069,18 +1065,16 @@ bool StorageReplicatedMergeTree::createTableIfNotExistsAttempt(const StorageMeta "of wrong zookeeper_path or because of logical error"); } -void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot, - const ZooKeeperRetriesInfo & zookeeper_retries_info, - QueryStatusPtr process_list_element) const +void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const { if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createReplica", log.load(), zookeeper_retries_info, process_list_element}; - retries_ctl.retryLoop([&] { createReplicaAttempt(metadata_snapshot, process_list_element); }); + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createReplica", log.load(), zookeeper_retries_info}; + retries_ctl.retryLoop([&] { createReplicaAttempt(metadata_snapshot, zookeeper_retries_info.query_status); }); } else { - createReplicaAttempt(metadata_snapshot, process_list_element); + createReplicaAttempt(metadata_snapshot, zookeeper_retries_info.query_status); } } @@ -1244,17 +1238,10 @@ ZooKeeperRetriesInfo StorageReplicatedMergeTree::getCreateQueryZooKeeperRetriesI return create_query_zookeeper_retries_info; } -QueryStatusPtr StorageReplicatedMergeTree::getCreateQueryStatus() const -{ - std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; - return create_query_status; -} - void StorageReplicatedMergeTree::clearCreateQueryZooKeeperRetriesInfo() { std::lock_guard lock{create_query_zookeeper_retries_info_mutex}; create_query_zookeeper_retries_info = {}; - create_query_status = {}; } @@ -1620,12 +1607,12 @@ bool StorageReplicatedMergeTree::removeTableNodesFromZooKeeper(zkutil::ZooKeeper */ bool StorageReplicatedMergeTree::checkTableStructure( const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, - const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const + const ZooKeeperRetriesInfo & zookeeper_retries_info) const { bool same_structure = false; if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkTableStructure", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::checkTableStructure", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { same_structure = checkTableStructureAttempt(zookeeper_prefix, metadata_snapshot, metadata_version, strict_check); }); } else @@ -1970,7 +1957,7 @@ bool StorageReplicatedMergeTree::checkPartsImpl(bool skip_sanity_checks) } -void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) +void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info) { String new_pinned_part_uuids_str; Coordination::Stat new_stat; @@ -1983,7 +1970,7 @@ void StorageReplicatedMergeTree::syncPinnedPartUUIDs(const ZooKeeperRetriesInfo if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::syncPinnedPartUUIDs", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::syncPinnedPartUUIDs", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { read_pinned_part_uuids(); }); } else @@ -2352,7 +2339,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) case LogEntry::ALTER_METADATA: return executeMetadataAlter(entry); case LogEntry::SYNC_PINNED_PART_UUIDS: - syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + syncPinnedPartUUIDs(/* zookeeper_retries_info = */ {}); return true; case LogEntry::CLONE_PART_FROM_SHARD: executeClonePartFromShard(entry); @@ -4501,7 +4488,7 @@ void StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String & part_n } -void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) +void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info) { if (!(*getSettings())[MergeTreeSetting::replicated_can_become_leader]) { @@ -4517,7 +4504,7 @@ void StorageReplicatedMergeTree::startBeingLeader(const ZooKeeperRetriesInfo & z if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::startBeingLeader", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::startBeingLeader", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { start_being_leader(); }); } else @@ -5396,10 +5383,10 @@ void StorageReplicatedMergeTree::startup() return; } - startupImpl(/* from_attach_thread */ false, getCreateQueryZooKeeperRetriesInfo(), getCreateQueryStatus()); + startupImpl(/* from_attach_thread */ false, getCreateQueryZooKeeperRetriesInfo()); } -void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) +void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info) { /// Do not start replication if ZooKeeper is not configured or there is no metadata in zookeeper if (!has_metadata_in_zookeeper.has_value() || !*has_metadata_in_zookeeper) @@ -5428,7 +5415,7 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread, const ZooK getContext()->getInterserverIOHandler().addEndpoint( data_parts_exchange_ptr->getId(getEndpointName()), data_parts_exchange_ptr); - startBeingLeader(zookeeper_retries_info, process_list_element); + startBeingLeader(zookeeper_retries_info); if (from_attach_thread) { @@ -6683,7 +6670,7 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition( return true; } -void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info) +void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info) { LOG_INFO(log, "Restoring replica metadata"); @@ -6726,14 +6713,14 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query LOG_INFO(log, "Moved all parts to detached/"); - const bool is_first_replica = createTableIfNotExists(metadata_snapshot, zookeeper_retries_info, query_status); + const bool is_first_replica = createTableIfNotExists(metadata_snapshot, zookeeper_retries_info); LOG_INFO(log, "Created initial ZK nodes, replica is first: {}", is_first_replica); if (!is_first_replica) - createReplica(metadata_snapshot, zookeeper_retries_info, query_status); + createReplica(metadata_snapshot, zookeeper_retries_info); - createNewZooKeeperNodes(zookeeper_retries_info, query_status); + createNewZooKeeperNodes(zookeeper_retries_info); LOG_INFO(log, "Created ZK nodes for table"); @@ -6745,7 +6732,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper(QueryStatusPtr query LOG_INFO(log, "Attached all partitions, starting table"); - startupImpl(/* from_attach_thread */ false, zookeeper_retries_info, query_status); + startupImpl(/* from_attach_thread */ false, zookeeper_retries_info); } void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) @@ -8034,8 +8021,8 @@ void StorageReplicatedMergeTree::forcefullyRemoveBrokenOutdatedPartFromZooKeeper String part_path = replica_path + "/parts/" + part_name; const auto & settings = getContext()->getSettingsRef(); ZooKeeperRetriesInfo retries_info{ - settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}; - ZooKeeperRetriesControl retries_ctl("outdatedPartExists", log.load(), retries_info, nullptr); + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms], nullptr}; + ZooKeeperRetriesControl retries_ctl("outdatedPartExists", log.load(), retries_info); retries_ctl.retryLoop([&]() { exists = getZooKeeper()->exists(part_path); }); if (!exists) @@ -8924,7 +8911,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( { /// Optimistic check that for compatible destination table structure. - checkTableStructure(to, getInMemoryMetadataPtr(), /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + checkTableStructure(to, getInMemoryMetadataPtr(), /* metadata_version = */ nullptr, /* strict_check = */ true, /* zookeeper_retries_info = */ {}); } PinnedPartUUIDs src_pins; @@ -9527,7 +9514,7 @@ String StorageReplicatedMergeTree::getTableSharedID() const { /// Can happen if table was partially initialized before drop by DatabaseCatalog if (table_shared_id == UUIDHelpers::Nil) - createTableSharedID(/* zookeeper_retries_info = */ {}, /* process_list_element = */ nullptr); + createTableSharedID(/* zookeeper_retries_info = */ {}); } else { @@ -9542,11 +9529,11 @@ std::map StorageReplicatedMergeTree::getUnfinishe return queue.getUnfinishedMutations(); } -void StorageReplicatedMergeTree::createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const +void StorageReplicatedMergeTree::createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info) const { if (zookeeper_retries_info.max_retries > 0) { - ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableSharedID", log.load(), zookeeper_retries_info, process_list_element}; + ZooKeeperRetriesControl retries_ctl{"StorageReplicatedMergeTree::createTableSharedID", log.load(), zookeeper_retries_info}; retries_ctl.retryLoop([&] { createTableSharedIDAttempt(); }); } else @@ -10836,7 +10823,7 @@ void StorageReplicatedMergeTree::backupData( bool exists = false; Strings mutation_ids; { - ZooKeeperRetriesControl retries_ctl("getMutations", log.load(), zookeeper_retries_info, nullptr); + ZooKeeperRetriesControl retries_ctl("getMutations", log.load(), zookeeper_retries_info); retries_ctl.retryLoop([&]() { if (!zookeeper || zookeeper->expired()) @@ -10855,7 +10842,7 @@ void StorageReplicatedMergeTree::backupData( bool mutation_id_exists = false; String mutation; - ZooKeeperRetriesControl retries_ctl("getMutation", log.load(), zookeeper_retries_info, nullptr); + ZooKeeperRetriesControl retries_ctl("getMutation", log.load(), zookeeper_retries_info); retries_ctl.retryLoop([&]() { if (!zookeeper || zookeeper->expired()) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index c6081d45bf4..f153ae91361 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -110,8 +110,7 @@ public: const MergingParams & merging_params_, std::unique_ptr settings_, bool need_check_structure, - const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_, - QueryStatusPtr create_query_status_); + const ZooKeeperRetriesInfo & create_query_zookeeper_retries_info_); void startup() override; @@ -316,7 +315,7 @@ public: /// Restores table metadata if ZooKeeper lost it. /// Used only on restarted readonly replicas (not checked). All active (Active) parts are moved to detached/ /// folder and attached. Parts in all other states are just moved to detached/ folder. - void restoreMetadataInZooKeeper(QueryStatusPtr query_status, const ZooKeeperRetriesInfo & zookeeper_retries_info); + void restoreMetadataInZooKeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info); /// Get throttler for replicated fetches ThrottlerPtr getFetchesThrottler() const @@ -428,7 +427,6 @@ private: const String replica_path; ZooKeeperRetriesInfo create_query_zookeeper_retries_info TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); - QueryStatusPtr create_query_status TSA_GUARDED_BY(create_query_zookeeper_retries_info_mutex); mutable std::mutex create_query_zookeeper_retries_info_mutex; /** /replicas/me/is_active. @@ -579,27 +577,26 @@ private: /** Creates the minimum set of nodes in ZooKeeper and create first replica. * Returns true if was created, false if exists. */ - bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const; bool createTableIfNotExistsAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** * Creates a replica in ZooKeeper and adds to the queue all that it takes to catch up with the rest of the replicas. */ - void createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createReplica(const StorageMetadataPtr & metadata_snapshot, const ZooKeeperRetriesInfo & zookeeper_retries_info) const; void createReplicaAttempt(const StorageMetadataPtr & metadata_snapshot, QueryStatusPtr process_list_element) const; /** Create nodes in the ZK, which must always be, but which might not exist when older versions of the server are running. */ - void createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createNewZooKeeperNodes(const ZooKeeperRetriesInfo & zookeeper_retries_info) const; void createNewZooKeeperNodesAttempt() const; /// Returns the ZooKeeper retries info specified for the CREATE TABLE query which is creating and starting this table right now. ZooKeeperRetriesInfo getCreateQueryZooKeeperRetriesInfo() const; - QueryStatusPtr getCreateQueryStatus() const; void clearCreateQueryZooKeeperRetriesInfo(); bool checkTableStructure(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check, - const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + const ZooKeeperRetriesInfo & zookeeper_retries_info) const; bool checkTableStructureAttempt(const String & zookeeper_prefix, const StorageMetadataPtr & metadata_snapshot, int32_t * metadata_version, bool strict_check) const; /// A part of ALTER: apply metadata changes only (data parts are altered separately). @@ -619,7 +616,7 @@ private: /// Synchronize the list of part uuids which are currently pinned. These should be sent to root query executor /// to be used for deduplication. - void syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); + void syncPinnedPartUUIDs(const ZooKeeperRetriesInfo & zookeeper_retries_info); /** Check that the part's checksum is the same as the checksum of the same part on some other replica. * If no one has such a part, nothing checks. @@ -722,7 +719,7 @@ private: /// Start being leader (if not disabled by setting). /// Since multi-leaders are allowed, it just sets is_leader flag. - void startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); + void startBeingLeader(const ZooKeeperRetriesInfo & zookeeper_retries_info); void stopBeingLeader(); /** Selects the parts to merge and writes to the log. @@ -939,7 +936,7 @@ private: /// Check granularity of already existing replicated table in zookeeper if it exists /// return true if it's fixed - bool checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + bool checkFixedGranularityInZookeeper(const ZooKeeperRetriesInfo & zookeeper_retries_info) const; /// Wait for timeout seconds mutation is finished on replicas void waitMutationToFinishOnReplicas( @@ -977,7 +974,7 @@ private: void createAndStoreFreezeMetadata(DiskPtr disk, DataPartPtr part, String backup_part_path) const override; // Create table id if needed - void createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element) const; + void createTableSharedID(const ZooKeeperRetriesInfo & zookeeper_retries_info) const; void createTableSharedIDAttempt() const; bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica); @@ -994,7 +991,7 @@ private: /// Or if node actually disappeared. bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override; - void startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info, QueryStatusPtr process_list_element); + void startupImpl(bool from_attach_thread, const ZooKeeperRetriesInfo & zookeeper_retries_info); std::vector getZookeeperZeroCopyLockPaths() const; static void dropZookeeperZeroCopyLockPaths(zkutil::ZooKeeperPtr zookeeper, diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index 000098af80d..1cd17cc0cd3 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -518,7 +518,8 @@ Chunk SystemZooKeeperSource::generate() ZooKeeperRetriesInfo retries_seetings( settings[Setting::insert_keeper_max_retries], settings[Setting::insert_keeper_retry_initial_backoff_ms], - settings[Setting::insert_keeper_retry_max_backoff_ms]); + settings[Setting::insert_keeper_retry_max_backoff_ms], + query_status); /// Handles reconnects when needed auto get_zookeeper = [&] () @@ -586,7 +587,7 @@ Chunk SystemZooKeeperSource::generate() } zkutil::ZooKeeper::MultiTryGetChildrenResponse list_responses; - ZooKeeperRetriesControl("", nullptr, retries_seetings, query_status).retryLoop( + ZooKeeperRetriesControl("", nullptr, retries_seetings).retryLoop( [&]() { list_responses = get_zookeeper()->tryGetChildren(paths_to_list); }); struct GetTask @@ -632,7 +633,7 @@ Chunk SystemZooKeeperSource::generate() } zkutil::ZooKeeper::MultiTryGetResponse get_responses; - ZooKeeperRetriesControl("", nullptr, retries_seetings, query_status).retryLoop( + ZooKeeperRetriesControl("", nullptr, retries_seetings).retryLoop( [&]() { get_responses = get_zookeeper()->tryGet(paths_to_get); }); /// Add children count to query total rows. We can not get total rows in advance, From d49e22cbae1eacbe0758f611a85b352a174563f3 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Dec 2024 17:50:32 +0000 Subject: [PATCH 12/38] Fix builds --- src/TableFunctions/ITableFunctionFileLike.cpp | 36 ------------------- src/TableFunctions/ITableFunctionFileLike.h | 36 ++++++++++++++++++- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index b591ea97e3d..159eb0a8f5d 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -88,42 +88,6 @@ void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & compression_method = checkAndGetLiteralArgument(args[3], "compression_method"); } -void ITableFunctionFileLike::updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context, bool with_structure) -{ - if (args.empty() || args.size() > getMaxNumberOfArguments()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); - - auto format_literal = std::make_shared(format); - auto structure_literal = std::make_shared(structure); - - for (auto & arg : args) - arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - - /// f(filename) - if (args.size() == 1) - { - args.push_back(format_literal); - if (with_structure) - args.push_back(structure_literal); - } - /// f(filename, format) - else if (args.size() == 2) - { - if (checkAndGetLiteralArgument(args[1], "format") == "auto") - args.back() = format_literal; - if (with_structure) - args.push_back(structure_literal); - } - /// f(filename, format, structure) or f(filename, format, structure, compression) or f(filename, format, compression) - else if (args.size() >= 3) - { - if (checkAndGetLiteralArgument(args[1], "format") == "auto") - args[1] = format_literal; - if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") - args[2] = structure_literal; - } -} - StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/, bool /*is_insert_query*/) const { ColumnsDescription columns; diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index f9b5e30e85e..ef8226ea62e 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -35,7 +35,41 @@ public: static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure); + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure) + { + if (args.empty() || args.size() > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); + + auto format_literal = std::make_shared(format); + auto structure_literal = std::make_shared(structure); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + + /// f(filename) + if (args.size() == 1) + { + args.push_back(format_literal); + if (with_structure) + args.push_back(structure_literal); + } + /// f(filename, format) + else if (args.size() == 2) + { + if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args.back() = format_literal; + if (with_structure) + args.push_back(structure_literal); + } + /// f(filename, format, structure) or f(filename, format, structure, compression) or f(filename, format, compression) + else if (args.size() >= 3) + { + if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args[1] = format_literal; + if (with_structure && checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; + } + } protected: From 136f5c482c01bf9129480bfe93a6d13da89a79e6 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:36:18 +0100 Subject: [PATCH 13/38] Fix style --- src/TableFunctions/ITableFunctionFileLike.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index ef8226ea62e..b7956a74ddc 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -6,6 +6,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + class ColumnsDescription; class Context; From df04a39b97f8f5c3ff3144f6aa55f20aceb04d9e Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 5 Dec 2024 13:16:48 +0000 Subject: [PATCH 14/38] Fix private --- src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 7 +++++-- .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index a4fa3a30d23..3c76ebe5d5e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -188,7 +188,7 @@ void MergeTreeReaderCompact::readData( auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr); + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name_in_storage], nullptr); auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); /// TODO: Avoid extra copying. @@ -231,6 +231,9 @@ void MergeTreeReaderCompact::readPrefix( { try { + if (deserialize_binary_bulk_state_map.contains(name_and_type.getNameInStorage())) + return; + ISerialization::DeserializeBinaryBulkSettings deserialize_settings; if (name_level_for_offsets.has_value()) @@ -253,7 +256,7 @@ void MergeTreeReaderCompact::readPrefix( deserialize_settings.getter = buffer_getter; deserialize_settings.object_and_dynamic_read_statistics = true; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.name], nullptr); + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.getNameInStorage()], nullptr); } catch (Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 0c3e67d5078..d72dd3f2269 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -25,6 +25,7 @@ try while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); + deserialize_binary_bulk_state_map.clear(); /// Use cache to avoid reading the column with the same name twice. /// It may happen if there are empty array Nested in the part. From 41c16b71d699642656c5b2d26ce0ef64fd840c1c Mon Sep 17 00:00:00 2001 From: Pavel Kruglov <48961922+Avogar@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:19:57 +0100 Subject: [PATCH 15/38] Fix build --- src/TableFunctions/ITableFunctionFileLike.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index b7956a74ddc..64c34311af2 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -41,7 +41,7 @@ public: static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } - static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &, bool with_structure) + static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr & context, bool with_structure) { if (args.empty() || args.size() > getMaxNumberOfArguments()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function, got {}", getMaxNumberOfArguments(), args.size()); From f2c328aa7a9c0ac7cc7a13f5d99201d46dc79361 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 6 Dec 2024 13:11:00 +0100 Subject: [PATCH 16/38] Revert "Merge pull request #72836 from ClickHouse/revert-72770-more_insistent_compress_in_memory_eng" This reverts commit f752c0b89d1a84fc92bfbb7fc510a092b5aa4c84, reversing changes made to 65d895a6dbd5f07383b871430f3cee5ce1b984a3. --- docs/en/engines/table-engines/special/memory.md | 2 ++ src/Columns/ColumnArray.cpp | 6 +++--- src/Columns/ColumnArray.h | 2 +- src/Columns/ColumnCompressed.cpp | 5 +++-- src/Columns/ColumnCompressed.h | 2 +- src/Columns/ColumnDecimal.cpp | 4 ++-- src/Columns/ColumnDecimal.h | 2 +- src/Columns/ColumnDynamic.cpp | 4 ++-- src/Columns/ColumnDynamic.h | 2 +- src/Columns/ColumnFixedString.cpp | 4 ++-- src/Columns/ColumnFixedString.h | 2 +- src/Columns/ColumnMap.cpp | 4 ++-- src/Columns/ColumnMap.h | 2 +- src/Columns/ColumnNullable.cpp | 6 +++--- src/Columns/ColumnNullable.h | 2 +- src/Columns/ColumnObject.cpp | 8 ++++---- src/Columns/ColumnObject.h | 2 +- src/Columns/ColumnSparse.cpp | 6 +++--- src/Columns/ColumnSparse.h | 2 +- src/Columns/ColumnString.cpp | 6 +++--- src/Columns/ColumnString.h | 2 +- src/Columns/ColumnTuple.cpp | 4 ++-- src/Columns/ColumnTuple.h | 2 +- src/Columns/ColumnVariant.cpp | 8 ++++---- src/Columns/ColumnVariant.h | 2 +- src/Columns/ColumnVector.cpp | 4 ++-- src/Columns/ColumnVector.h | 2 +- src/Columns/IColumn.h | 3 ++- src/Core/Block.cpp | 2 +- src/Interpreters/Cache/QueryCache.cpp | 2 +- src/Storages/StorageMemory.cpp | 7 +++---- 31 files changed, 57 insertions(+), 54 deletions(-) diff --git a/docs/en/engines/table-engines/special/memory.md b/docs/en/engines/table-engines/special/memory.md index f28157ebde2..3eb3e617ff9 100644 --- a/docs/en/engines/table-engines/special/memory.md +++ b/docs/en/engines/table-engines/special/memory.md @@ -36,6 +36,8 @@ Upper and lower bounds can be specified to limit Memory engine table size, effec - Requires `max_rows_to_keep` - `max_rows_to_keep` — Maximum rows to keep within memory table where oldest rows are deleted on each insertion (i.e circular buffer). Max rows can exceed the stated limit if the oldest batch of rows to remove falls under the `min_rows_to_keep` limit when adding a large block. - Default value: `0` +- `compress` - Whether to compress data in memory. + - Default value: `false` ## Usage {#usage} diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 3f88ca93a97..013821db2c9 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1024,10 +1024,10 @@ void ColumnArray::updatePermutationWithCollation(const Collator & collator, Perm DefaultPartialSort()); } -ColumnPtr ColumnArray::compress() const +ColumnPtr ColumnArray::compress(bool force_compression) const { - ColumnPtr data_compressed = data->compress(); - ColumnPtr offsets_compressed = offsets->compress(); + ColumnPtr data_compressed = data->compress(force_compression); + ColumnPtr offsets_compressed = offsets->compress(force_compression); size_t byte_size = data_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index a66f9041213..dee6ae931f2 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -159,7 +159,7 @@ public: /// For example, `getDataInRange(0, size())` is the same as `getDataPtr()->clone()`. MutableColumnPtr getDataInRange(size_t start, size_t length) const; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnCompressed.cpp b/src/Columns/ColumnCompressed.cpp index 3bdc514d6d8..adb2a5f391d 100644 --- a/src/Columns/ColumnCompressed.cpp +++ b/src/Columns/ColumnCompressed.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes } -std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool always_compress) +std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, size_t data_size, bool force_compression) { size_t max_dest_size = LZ4_COMPRESSBOUND(data_size); @@ -35,7 +35,8 @@ std::shared_ptr> ColumnCompressed::compressBuffer(const void * data, si throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress column"); /// If compression is inefficient. - if (!always_compress && static_cast(compressed_size) * 2 > data_size) + const size_t threshold = force_compression ? 1 : 2; + if (static_cast(compressed_size) * threshold > data_size) return {}; /// Shrink to fit. diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index c4270e8216b..b030e762acd 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -72,7 +72,7 @@ public: /// If data is not worth to be compressed and not 'always_compress' - returns nullptr. /// Note: shared_ptr is to allow to be captured by std::function. - static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool always_compress); + static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); static void decompressBuffer( const void * compressed_data, void * decompressed_data, size_t compressed_size, size_t decompressed_size); diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 73366150e7d..c286c54198a 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -478,7 +478,7 @@ ColumnPtr ColumnDecimal::replicate(const IColumn::Offsets & offsets) const } template -ColumnPtr ColumnDecimal::compress() const +ColumnPtr ColumnDecimal::compress(bool force_compression) const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -487,7 +487,7 @@ ColumnPtr ColumnDecimal::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 690549e4a56..3e5c189b731 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -140,7 +140,7 @@ public: return false; } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void insertValue(const T value) { data.push_back(value); } Container & getData() { return data; } diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index a4c932eafdd..2d05701c57b 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -991,9 +991,9 @@ void ColumnDynamic::updatePermutation(IColumn::PermutationSortDirection directio updatePermutationImpl(limit, res, equal_ranges, ComparatorDescendingStable(*this, nan_direction_hint), comparator_equal, DefaultSort(), DefaultPartialSort()); } -ColumnPtr ColumnDynamic::compress() const +ColumnPtr ColumnDynamic::compress(bool force_compression) const { - ColumnPtr variant_compressed = variant_column_ptr->compress(); + ColumnPtr variant_compressed = variant_column_ptr->compress(force_compression); size_t byte_size = variant_compressed->byteSize(); return ColumnCompressed::create(size(), byte_size, [my_variant_compressed = std::move(variant_compressed), my_variant_info = variant_info, my_max_dynamic_types = max_dynamic_types, my_global_max_dynamic_types = global_max_dynamic_types, my_statistics = statistics]() mutable diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index bdbad99519f..093aaaf2793 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -335,7 +335,7 @@ public: return false; } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; double getRatioOfDefaultRows(double sample_ratio) const override { diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 04e894ee5ab..f076f904768 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -419,7 +419,7 @@ void ColumnFixedString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnFixedString::compress() const +ColumnPtr ColumnFixedString::compress(bool force_compression) const { size_t source_size = chars.size(); @@ -427,7 +427,7 @@ ColumnPtr ColumnFixedString::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(chars.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 8cf0a6a57da..f55fb60a976 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -175,7 +175,7 @@ public: ColumnPtr replicate(const Offsets & offsets) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void reserve(size_t size) override { diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index a5511dfeeb4..fb9c8c9fbaf 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -352,9 +352,9 @@ bool ColumnMap::dynamicStructureEquals(const IColumn & rhs) const return false; } -ColumnPtr ColumnMap::compress() const +ColumnPtr ColumnMap::compress(bool force_compression) const { - auto compressed = nested->compress(); + auto compressed = nested->compress(force_compression); const auto byte_size = compressed->byteSize(); /// The order of evaluation of function arguments is unspecified /// and could cause interacting with object in moved-from state diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h index 8dfa5bb5845..31404a3e152 100644 --- a/src/Columns/ColumnMap.h +++ b/src/Columns/ColumnMap.h @@ -120,7 +120,7 @@ public: const ColumnTuple & getNestedData() const { return assert_cast(getNestedColumn().getData()); } ColumnTuple & getNestedData() { return assert_cast(getNestedColumn().getData()); } - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; bool hasDynamicStructure() const override { return nested->hasDynamicStructure(); } bool dynamicStructureEquals(const IColumn & rhs) const override; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index 6e8bd3fc70c..640550fcf9a 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -773,10 +773,10 @@ void ColumnNullable::protect() getNullMapColumn().protect(); } -ColumnPtr ColumnNullable::compress() const +ColumnPtr ColumnNullable::compress(bool force_compression) const { - ColumnPtr nested_compressed = nested_column->compress(); - ColumnPtr null_map_compressed = null_map->compress(); + ColumnPtr nested_compressed = nested_column->compress(force_compression); + ColumnPtr null_map_compressed = null_map->compress(force_compression); size_t byte_size = nested_column->byteSize() + null_map->byteSize(); diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h index 32ce66c5965..3a0be008cc2 100644 --- a/src/Columns/ColumnNullable.h +++ b/src/Columns/ColumnNullable.h @@ -141,7 +141,7 @@ public: // Special function for nullable minmax index void getExtremesNullLast(Field & min, Field & max) const; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index dae7b4c36b9..f110e8b442b 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1225,14 +1225,14 @@ bool ColumnObject::structureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnObject::compress() const +ColumnPtr ColumnObject::compress(bool force_compression) const { std::unordered_map compressed_typed_paths; compressed_typed_paths.reserve(typed_paths.size()); size_t byte_size = 0; for (const auto & [path, column] : typed_paths) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed_typed_paths[path] = std::move(compressed_column); } @@ -1241,12 +1241,12 @@ ColumnPtr ColumnObject::compress() const compressed_dynamic_paths.reserve(dynamic_paths_ptrs.size()); for (const auto & [path, column] : dynamic_paths_ptrs) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed_dynamic_paths[path] = std::move(compressed_column); } - auto compressed_shared_data = shared_data->compress(); + auto compressed_shared_data = shared_data->compress(force_compression); byte_size += compressed_shared_data->byteSize(); auto decompress = diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h index 7b8a381d571..3160b66cd20 100644 --- a/src/Columns/ColumnObject.h +++ b/src/Columns/ColumnObject.h @@ -171,7 +171,7 @@ public: bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp index a0e47e65fc6..b7d82ed8a09 100644 --- a/src/Columns/ColumnSparse.cpp +++ b/src/Columns/ColumnSparse.cpp @@ -774,10 +774,10 @@ UInt64 ColumnSparse::getNumberOfDefaultRows() const return _size - offsets->size(); } -ColumnPtr ColumnSparse::compress() const +ColumnPtr ColumnSparse::compress(bool force_compression) const { - auto values_compressed = values->compress(); - auto offsets_compressed = offsets->compress(); + auto values_compressed = values->compress(force_compression); + auto offsets_compressed = offsets->compress(force_compression); size_t byte_size = values_compressed->byteSize() + offsets_compressed->byteSize(); diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h index 619dce63c1e..f95752cd546 100644 --- a/src/Columns/ColumnSparse.h +++ b/src/Columns/ColumnSparse.h @@ -147,7 +147,7 @@ public: double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; ColumnCheckpointPtr getCheckpoint() const override; void updateCheckpoint(ColumnCheckpoint & checkpoint) const override; diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 9569e9ec252..4bdc253bfc4 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -628,7 +628,7 @@ void ColumnString::getExtremes(Field & min, Field & max) const get(max_idx, max); } -ColumnPtr ColumnString::compress() const +ColumnPtr ColumnString::compress(bool force_compression) const { const size_t source_chars_size = chars.size(); const size_t source_offsets_elements = offsets.size(); @@ -638,13 +638,13 @@ ColumnPtr ColumnString::compress() const if (source_chars_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, false); + auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, force_compression); /// Return original column if not compressible. if (!chars_compressed) return ColumnCompressed::wrap(this->getPtr()); - auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, true); + auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); const size_t chars_compressed_size = chars_compressed->size(); const size_t offsets_compressed_size = offsets_compressed->size(); diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 062315219b5..4bf24217383 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -272,7 +272,7 @@ public: ColumnPtr replicate(const Offsets & replicate_offsets) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void reserve(size_t n) override; size_t capacity() const override; diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 28e5f03cc3c..9bb377f56ae 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -796,7 +796,7 @@ void ColumnTuple::takeDynamicStructureFromSourceColumns(const Columns & source_c } -ColumnPtr ColumnTuple::compress() const +ColumnPtr ColumnTuple::compress(bool force_compression) const { if (columns.empty()) { @@ -812,7 +812,7 @@ ColumnPtr ColumnTuple::compress() const compressed.reserve(columns.size()); for (const auto & column : columns) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(force_compression); byte_size += compressed_column->byteSize(); compressed.emplace_back(std::move(compressed_column)); } diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index d5eee911edc..b8b3697b84d 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -125,7 +125,7 @@ public: void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; bool isCollationSupported() const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; void finalize() override; bool isFinalized() const override; diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index 2fa59b8e33c..38d3bac3c10 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -1426,16 +1426,16 @@ bool ColumnVariant::dynamicStructureEquals(const IColumn & rhs) const return true; } -ColumnPtr ColumnVariant::compress() const +ColumnPtr ColumnVariant::compress(bool force_compression) const { - ColumnPtr local_discriminators_compressed = local_discriminators->compress(); - ColumnPtr offsets_compressed = offsets->compress(); + ColumnPtr local_discriminators_compressed = local_discriminators->compress(force_compression); + ColumnPtr offsets_compressed = offsets->compress(force_compression); size_t byte_size = local_discriminators_compressed->byteSize() + offsets_compressed->byteSize(); Columns compressed; compressed.reserve(variants.size()); for (const auto & variant : variants) { - auto compressed_variant = variant->compress(); + auto compressed_variant = variant->compress(force_compression); byte_size += compressed_variant->byteSize(); compressed.emplace_back(std::move(compressed_variant)); } diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index a68a961169c..c7e37517004 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -254,7 +254,7 @@ public: void forEachSubcolumn(MutableColumnCallback callback) override; void forEachSubcolumnRecursively(RecursiveMutableColumnCallback callback) override; bool structureEquals(const IColumn & rhs) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; double getRatioOfDefaultRows(double sample_ratio) const override; UInt64 getNumberOfDefaultRows() const override; void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 3c7727f37c4..62f6c23c4f8 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -951,7 +951,7 @@ void ColumnVector::getExtremes(Field & min, Field & max) const } template -ColumnPtr ColumnVector::compress() const +ColumnPtr ColumnVector::compress(bool force_compression) const { const size_t data_size = data.size(); const size_t source_size = data_size * sizeof(T); @@ -960,7 +960,7 @@ ColumnPtr ColumnVector::compress() const if (source_size < 4096) /// A wild guess. return ColumnCompressed::wrap(this->getPtr()); - auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, false); + auto compressed = ColumnCompressed::compressBuffer(data.data(), source_size, force_compression); if (!compressed) return ColumnCompressed::wrap(this->getPtr()); diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 5247bfdf972..e5ece863a3b 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -287,7 +287,7 @@ public: ColumnPtr createWithOffsets(const IColumn::Offsets & offsets, const ColumnConst & column_with_default_value, size_t total_rows, size_t shift) const override; - ColumnPtr compress() const override; + ColumnPtr compress(bool force_compression) const override; /// Replace elements that match the filter with zeroes. If inverted replaces not matched elements. void applyZeroMap(const IColumn::Filter & filt, bool inverted = false); diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 9d1b42d2bc1..e2099ac34b9 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -601,7 +601,8 @@ public: /// Compress column in memory to some representation that allows to decompress it back. /// Return itself if compression is not applicable for this column type. - [[nodiscard]] virtual Ptr compress() const + /// The flag `force_compression` indicates that compression should be performed even if it's not efficient (if only compression factor < 1). + [[nodiscard]] virtual Ptr compress([[maybe_unused]] bool force_compression) const { /// No compression by default. return getPtr(); diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 02176a6b77a..0efb4596dcd 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -608,7 +608,7 @@ Block Block::compress() const size_t num_columns = data.size(); Columns new_columns(num_columns); for (size_t i = 0; i < num_columns; ++i) - new_columns[i] = data[i].column->compress(); + new_columns[i] = data[i].column->compress(/*force_compression=*/false); return cloneWithColumns(new_columns); } diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 7dbee567c5b..de3d720fc35 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -469,7 +469,7 @@ void QueryCache::Writer::finalizeWrite() Columns compressed_columns; for (const auto & column : columns) { - auto compressed_column = column->compress(); + auto compressed_column = column->compress(/*force_compression=*/false); compressed_columns.push_back(compressed_column); } Chunk compressed_chunk(compressed_columns, chunk.getNumRows()); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index b56d6d98680..fcc24a06881 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -91,8 +91,7 @@ public: { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); - + compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); new_blocks.push_back(std::move(compressed_block)); } else @@ -259,7 +258,7 @@ void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context { if ((*memory_settings)[MemorySetting::compress]) for (auto & elem : block) - elem.column = elem.column->compress(); + elem.column = elem.column->compress(/*force_compression=*/true); out.push_back(block); } @@ -574,7 +573,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat { Block compressed_block; for (const auto & elem : block) - compressed_block.insert({ elem.column->compress(), elem.type, elem.name }); + compressed_block.insert({elem.column->compress(/*force_compression=*/true), elem.type, elem.name}); new_blocks.push_back(std::move(compressed_block)); } From 7c7b2314765033493f326df4dafc462c25bcdd24 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 6 Dec 2024 13:58:35 +0000 Subject: [PATCH 17/38] Update test --- tests/performance/compact_part_subcolumns.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/performance/compact_part_subcolumns.xml b/tests/performance/compact_part_subcolumns.xml index 234b33ab8a7..d0544458589 100644 --- a/tests/performance/compact_part_subcolumns.xml +++ b/tests/performance/compact_part_subcolumns.xml @@ -3,9 +3,6 @@ 1 - - CREATE TABLE t_json (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=100000000000 INSERT INTO t_json SELECT toJSONString(map(number % 10, repeat('a', number % 100))) FROM numbers(10000000) SELECT data.k0, data.k1, data.k2, data.k3, data.k4, data.k5, data.k6, data.k7, data.k8, data.k9 FROM t_json FORMAT Null From 82da99cfd317f4087696548f5173736eb6eba60f Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 6 Dec 2024 15:34:39 +0100 Subject: [PATCH 18/38] fix --- src/Columns/ColumnCompressed.h | 2 +- src/Columns/ColumnString.cpp | 43 ++++++++++---- src/Columns/ColumnString.h | 2 + src/Columns/tests/gtest_column_string.cpp | 71 +++++++++++++++++++++++ 4 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 src/Columns/tests/gtest_column_string.cpp diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index b030e762acd..7d7970cce8a 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -70,7 +70,7 @@ public: /// Helper methods for compression. - /// If data is not worth to be compressed and not 'always_compress' - returns nullptr. + /// If data is not worth to be compressed and not `force_compression` - returns nullptr. /// Note: shared_ptr is to allow to be captured by std::function. static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 4bdc253bfc4..3c73a005673 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -635,26 +635,39 @@ ColumnPtr ColumnString::compress(bool force_compression) const const size_t source_offsets_size = source_offsets_elements * sizeof(Offset); /// Don't compress small blocks. - if (source_chars_size < 4096) /// A wild guess. + if (source_chars_size < min_size_to_compress) + { return ColumnCompressed::wrap(this->getPtr()); + } auto chars_compressed = ColumnCompressed::compressBuffer(chars.data(), source_chars_size, force_compression); /// Return original column if not compressible. if (!chars_compressed) + { return ColumnCompressed::wrap(this->getPtr()); + } auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); + const bool offsets_were_compressed = !!offsets_compressed; + + /// Offsets are not compressible. Use the source data. + if (!offsets_compressed) + { + offsets_compressed = std::make_shared>(source_offsets_size); + memcpy(offsets_compressed->data(), offsets.data(), source_offsets_size); + } const size_t chars_compressed_size = chars_compressed->size(); const size_t offsets_compressed_size = offsets_compressed->size(); - return ColumnCompressed::create(source_offsets_elements, chars_compressed_size + offsets_compressed_size, - [ - my_chars_compressed = std::move(chars_compressed), - my_offsets_compressed = std::move(offsets_compressed), - source_chars_size, - source_offsets_elements - ] + return ColumnCompressed::create( + source_offsets_elements, + chars_compressed_size + offsets_compressed_size, + [my_chars_compressed = std::move(chars_compressed), + my_offsets_compressed = std::move(offsets_compressed), + source_chars_size, + source_offsets_elements, + offsets_were_compressed] { auto res = ColumnString::create(); @@ -664,8 +677,18 @@ ColumnPtr ColumnString::compress(bool force_compression) const ColumnCompressed::decompressBuffer( my_chars_compressed->data(), res->getChars().data(), my_chars_compressed->size(), source_chars_size); - ColumnCompressed::decompressBuffer( - my_offsets_compressed->data(), res->getOffsets().data(), my_offsets_compressed->size(), source_offsets_elements * sizeof(Offset)); + if (offsets_were_compressed) + { + ColumnCompressed::decompressBuffer( + my_offsets_compressed->data(), + res->getOffsets().data(), + my_offsets_compressed->size(), + source_offsets_elements * sizeof(Offset)); + } + else + { + memcpy(res->getOffsets().data(), my_offsets_compressed->data(), my_offsets_compressed->size()); + } return res; }); diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 4bf24217383..245164ca31b 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -29,6 +29,8 @@ public: using Char = UInt8; using Chars = PaddedPODArray; + static constexpr size_t min_size_to_compress = 4096; + private: friend class COWHelper, ColumnString>; diff --git a/src/Columns/tests/gtest_column_string.cpp b/src/Columns/tests/gtest_column_string.cpp new file mode 100644 index 00000000000..13a29616802 --- /dev/null +++ b/src/Columns/tests/gtest_column_string.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#include +#include + +using namespace DB; + +static pcg64 rng(randomSeed()); + +constexpr size_t bytes_per_string = sizeof(size_t) + 1; +/// Column should have enough bytes to be compressed +constexpr size_t column_size = ColumnString::min_size_to_compress / bytes_per_string + 42; + +TEST(ColumnString, Incompressible) +{ + auto col = ColumnString::create(); + auto & chars = col->getChars(); + auto & offsets = col->getOffsets(); + chars.resize(column_size * bytes_per_string); + for (size_t i = 0; i < column_size; ++i) + { + auto value = rng(); + memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); + chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + offsets.push_back((i + 1) * bytes_per_string); + } + + auto compressed = col->compress(true); + auto decompressed = compressed->decompress(); + ASSERT_EQ(decompressed.get(), col.get()); +} + +TEST(ColumnString, CompressibleCharsAndIncompressibleOffsets) +{ + auto col = ColumnString::create(); + auto & chars = col->getChars(); + auto & offsets = col->getOffsets(); + chars.resize(column_size * bytes_per_string); + for (size_t i = 0; i < column_size; ++i) + { + static const size_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); + chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + } + offsets.push_back(chars.size()); + + auto compressed = col->compress(true); + auto decompressed = compressed->decompress(); + ASSERT_NE(decompressed.get(), col.get()); +} + +TEST(ColumnString, CompressibleCharsAndCompressibleOffsets) +{ + auto col = ColumnString::create(); + auto & chars = col->getChars(); + auto & offsets = col->getOffsets(); + chars.resize(column_size * bytes_per_string); + for (size_t i = 0; i < column_size; ++i) + { + static const size_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); + chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + offsets.push_back((i + 1) * bytes_per_string); + } + + auto compressed = col->compress(true); + auto decompressed = compressed->decompress(); + ASSERT_NE(decompressed.get(), col.get()); +} From 5a9062d1200c9f9c3bae5c9601f4ab4805be59b8 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 6 Dec 2024 16:21:07 +0100 Subject: [PATCH 19/38] better --- src/Columns/tests/gtest_column_string.cpp | 37 +++++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Columns/tests/gtest_column_string.cpp b/src/Columns/tests/gtest_column_string.cpp index 13a29616802..4a0de2b5515 100644 --- a/src/Columns/tests/gtest_column_string.cpp +++ b/src/Columns/tests/gtest_column_string.cpp @@ -9,7 +9,7 @@ using namespace DB; static pcg64 rng(randomSeed()); -constexpr size_t bytes_per_string = sizeof(size_t) + 1; +constexpr size_t bytes_per_string = sizeof(uint64_t) + 1; /// Column should have enough bytes to be compressed constexpr size_t column_size = ColumnString::min_size_to_compress / bytes_per_string + 42; @@ -21,15 +21,20 @@ TEST(ColumnString, Incompressible) chars.resize(column_size * bytes_per_string); for (size_t i = 0; i < column_size; ++i) { - auto value = rng(); - memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); - chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + const uint64_t value = rng(); + memcpy(&chars[i * bytes_per_string], &value, sizeof(uint64_t)); + chars[i * bytes_per_string + sizeof(uint64_t)] = '\0'; offsets.push_back((i + 1) * bytes_per_string); } auto compressed = col->compress(true); auto decompressed = compressed->decompress(); + // When column is incompressible, we return the original column wrapped in CompressedColumn ASSERT_EQ(decompressed.get(), col.get()); + ASSERT_EQ(compressed->size(), col->size()); + ASSERT_EQ(compressed->allocatedBytes(), col->allocatedBytes()); + ASSERT_EQ(decompressed->size(), col->size()); + ASSERT_EQ(decompressed->allocatedBytes(), col->allocatedBytes()); } TEST(ColumnString, CompressibleCharsAndIncompressibleOffsets) @@ -40,15 +45,21 @@ TEST(ColumnString, CompressibleCharsAndIncompressibleOffsets) chars.resize(column_size * bytes_per_string); for (size_t i = 0; i < column_size; ++i) { - static const size_t value = 42; - memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); - chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + static const uint64_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(uint64_t)); + chars[i * bytes_per_string + sizeof(uint64_t)] = '\0'; } offsets.push_back(chars.size()); auto compressed = col->compress(true); auto decompressed = compressed->decompress(); + // For actually compressed column only compressed `chars` and `offsets` arrays are stored. + // Upon decompression, a new column is created. ASSERT_NE(decompressed.get(), col.get()); + ASSERT_EQ(compressed->size(), col->size()); + ASSERT_LE(compressed->allocatedBytes(), col->allocatedBytes()); + ASSERT_EQ(decompressed->size(), col->size()); + ASSERT_LE(decompressed->allocatedBytes(), col->allocatedBytes()); } TEST(ColumnString, CompressibleCharsAndCompressibleOffsets) @@ -59,13 +70,19 @@ TEST(ColumnString, CompressibleCharsAndCompressibleOffsets) chars.resize(column_size * bytes_per_string); for (size_t i = 0; i < column_size; ++i) { - static const size_t value = 42; - memcpy(&chars[i * bytes_per_string], &value, sizeof(size_t)); - chars[i * bytes_per_string + sizeof(size_t)] = '\0'; + static const uint64_t value = 42; + memcpy(&chars[i * bytes_per_string], &value, sizeof(uint64_t)); + chars[i * bytes_per_string + sizeof(uint64_t)] = '\0'; offsets.push_back((i + 1) * bytes_per_string); } auto compressed = col->compress(true); auto decompressed = compressed->decompress(); + // For actually compressed column only compressed `chars` and `offsets` arrays are stored. + // Upon decompression, a new column is created. ASSERT_NE(decompressed.get(), col.get()); + ASSERT_EQ(compressed->size(), col->size()); + ASSERT_LE(compressed->allocatedBytes(), col->allocatedBytes()); + ASSERT_EQ(decompressed->size(), col->size()); + ASSERT_LE(decompressed->allocatedBytes(), col->allocatedBytes()); } From 71849444fc165975775c898715c667316e492c4f Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 6 Dec 2024 18:11:12 +0000 Subject: [PATCH 20/38] Fix tests --- src/Storages/MergeTree/IMergeTreeReader.h | 4 ++- .../MergeTree/MergeTreeReaderCompact.cpp | 27 +++++++++++-------- .../MergeTreeReaderCompactSingleBuffer.cpp | 1 + 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h index c68617d3995..a80467e630e 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.h +++ b/src/Storages/MergeTree/IMergeTreeReader.h @@ -81,8 +81,10 @@ protected: /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; - /// Stores states for IDataType::deserializeBinaryBulk + /// Stores states for IDataType::deserializeBinaryBulk for regular columns. DeserializeBinaryBulkStateMap deserialize_binary_bulk_state_map; + /// The same as above, but for subcolumns. + DeserializeBinaryBulkStateMap deserialize_binary_bulk_state_map_for_subcolumns; /// Actual column names and types of columns in part, /// which may differ from table metadata. diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 3c76ebe5d5e..685b37a6bde 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -188,7 +188,7 @@ void MergeTreeReaderCompact::readData( auto serialization = getSerializationInPart({name_in_storage, type_in_storage}); ColumnPtr temp_column = type_in_storage->createColumn(*serialization); - serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name_in_storage], nullptr); + serialization->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map_for_subcolumns[name_in_storage], nullptr); auto subcolumn = type_in_storage->getSubcolumn(name_and_type.getSubcolumnName(), temp_column); /// TODO: Avoid extra copying. @@ -231,9 +231,6 @@ void MergeTreeReaderCompact::readPrefix( { try { - if (deserialize_binary_bulk_state_map.contains(name_and_type.getNameInStorage())) - return; - ISerialization::DeserializeBinaryBulkSettings deserialize_settings; if (name_level_for_offsets.has_value()) @@ -248,15 +245,23 @@ void MergeTreeReaderCompact::readPrefix( serialization_for_prefix->deserializeBinaryBulkStatePrefix(deserialize_settings, state_for_prefix, nullptr); } - SerializationPtr serialization; - if (name_and_type.isSubcolumn()) - serialization = getSerializationInPart({name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}); - else - serialization = getSerializationInPart(name_and_type); - deserialize_settings.getter = buffer_getter; deserialize_settings.object_and_dynamic_read_statistics = true; - serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.getNameInStorage()], nullptr); + + if (name_and_type.isSubcolumn()) + { + /// For subcolumns of the same column we need to deserialize prefix only once. + if (deserialize_binary_bulk_state_map_for_subcolumns.contains(name_and_type.getNameInStorage())) + return; + + auto serialization = getSerializationInPart({name_and_type.getNameInStorage(), name_and_type.getTypeInStorage()}); + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map_for_subcolumns[name_and_type.getNameInStorage()], nullptr); + } + else + { + auto serialization = getSerializationInPart(name_and_type); + serialization->deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name_and_type.getNameInStorage()], nullptr); + } } catch (Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index d72dd3f2269..4b59540b993 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -26,6 +26,7 @@ try { size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark); deserialize_binary_bulk_state_map.clear(); + deserialize_binary_bulk_state_map_for_subcolumns.clear(); /// Use cache to avoid reading the column with the same name twice. /// It may happen if there are empty array Nested in the part. From d5383d4d1b9f1dbd120875cce30b4113b84f1390 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 6 Dec 2024 18:31:56 +0000 Subject: [PATCH 21/38] Fix build --- src/TableFunctions/ITableFunctionFileLike.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 64c34311af2..321bdcf0d3b 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -4,6 +4,9 @@ #include "Core/Names.h" #include "Parsers/IAST_fwd.h" +#include +#include + namespace DB { From 646d44e30ddcfc7d4193854c765a93b2853aa887 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Mon, 9 Dec 2024 15:14:58 +0100 Subject: [PATCH 22/38] fix comment --- src/Columns/ColumnCompressed.h | 4 +++- src/Columns/ColumnString.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h index 7d7970cce8a..48f3717291b 100644 --- a/src/Columns/ColumnCompressed.h +++ b/src/Columns/ColumnCompressed.h @@ -70,7 +70,9 @@ public: /// Helper methods for compression. - /// If data is not worth to be compressed and not `force_compression` - returns nullptr. + /// If data is not worth to be compressed - returns nullptr. + /// By default it requires that compressed data is at least 50% smaller than original. + /// With `force_compression` set to true, it requires compressed data to be not larger than the source data. /// Note: shared_ptr is to allow to be captured by std::function. static std::shared_ptr> compressBuffer(const void * data, size_t data_size, bool force_compression); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 3c73a005673..4785f0ce28d 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -648,7 +648,7 @@ ColumnPtr ColumnString::compress(bool force_compression) const return ColumnCompressed::wrap(this->getPtr()); } - auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, /*force_compression=*/true); + auto offsets_compressed = ColumnCompressed::compressBuffer(offsets.data(), source_offsets_size, force_compression); const bool offsets_were_compressed = !!offsets_compressed; /// Offsets are not compressible. Use the source data. From 6da4d19fe0fdbbe574f94638cfb1c3180b19d488 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 9 Dec 2024 15:26:40 +0000 Subject: [PATCH 23/38] Fix tests --- src/Storages/StorageURL.cpp | 7 ++++--- src/Storages/StorageURL.h | 2 +- tests/queries/0_stateless/02968_url_args.reference | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 9b7b7227ad2..5ac37cbd418 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1488,7 +1488,7 @@ FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Argum } size_t StorageURL::evalArgsAndCollectHeaders( - ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context) + ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context, bool evaluate_arguments) { ASTs::iterator headers_it = url_function_args.end(); @@ -1542,7 +1542,8 @@ size_t StorageURL::evalArgsAndCollectHeaders( if (headers_ast_function && headers_ast_function->name == "equals") continue; - (*arg_it) = evaluateConstantExpressionOrIdentifierAsLiteral((*arg_it), context); + if (evaluate_arguments) + (*arg_it) = evaluateConstantExpressionOrIdentifierAsLiteral((*arg_it), context); } if (headers_it == url_function_args.end()) @@ -1584,7 +1585,7 @@ StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, const Contex if (auto named_collection = tryGetNamedCollectionWithOverrides(args, local_context)) { StorageURL::processNamedCollectionResult(configuration, *named_collection); - evalArgsAndCollectHeaders(args, configuration.headers, local_context); + evalArgsAndCollectHeaders(args, configuration.headers, local_context, false); } else { diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 7df5b90653d..b71f0cd68f8 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -322,7 +322,7 @@ public: /// Does evaluateConstantExpressionOrIdentifierAsLiteral() on all arguments. /// If `headers(...)` argument is present, parses it and moves it to the end of the array. /// Returns number of arguments excluding `headers(...)`. - static size_t evalArgsAndCollectHeaders(ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context); + static size_t evalArgsAndCollectHeaders(ASTs & url_function_args, HTTPHeaderEntries & header_entries, const ContextPtr & context, bool evaluate_arguments = true); static void processNamedCollectionResult(Configuration & configuration, const NamedCollection & collection); }; diff --git a/tests/queries/0_stateless/02968_url_args.reference b/tests/queries/0_stateless/02968_url_args.reference index e7e9e2c0d94..3c51a5edf81 100644 --- a/tests/queries/0_stateless/02968_url_args.reference +++ b/tests/queries/0_stateless/02968_url_args.reference @@ -1,7 +1,7 @@ CREATE TABLE default.a\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) CREATE TABLE default.b\n(\n `x` Int64\n)\nENGINE = URL(\'https://example.com/\', \'CSV\', headers()) CREATE TABLE default.c\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', \'CSV\', headers(\'foo\' = \'[HIDDEN]\')) -CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'[HIDDEN]\')) +CREATE TABLE default.d\n(\n `x` Int64\n)\nENGINE = S3(\'https://example.s3.amazonaws.com/a.csv\', \'NOSIGN\', headers(\'foo\' = \'bar\'), \'CSV\') CREATE VIEW default.e\n(\n `x` Int64\n)\nAS SELECT count()\nFROM url(\'https://example.com/\', CSV, headers(\'foo\' = \'[HIDDEN]\', \'a\' = \'[HIDDEN]\')) CREATE VIEW default.f\n(\n `x` Int64\n)\nAS SELECT count()\nFROM url(\'https://example.com/\', CSV, headers()) CREATE VIEW default.g\n(\n `x` Int64\n)\nAS SELECT count()\nFROM s3(\'https://example.s3.amazonaws.com/a.csv\', CSV, headers(\'foo\' = \'[HIDDEN]\')) From c9c8c9d26708da0b0648aa4aa94bc1277228e021 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Tue, 30 Jul 2024 16:35:29 +0200 Subject: [PATCH 24/38] Tests: add RMV tests --- .../sql-reference/statements/create/view.md | 11 +- tests/integration/helpers/client.py | 25 +- tests/integration/helpers/cluster.py | 20 +- .../configs/config.xml | 2 + .../configs/remote_servers.xml | 27 + .../configs/settings.xml | 8 + .../schedule_model.py | 143 + .../test_refreshable_mat_view/test.py | 2530 +++++++++++++++++ .../test_schedule_model.py | 90 + .../__init__.py | 0 .../configs/settings.xml | 16 + 11 files changed, 2860 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_refreshable_mat_view/configs/config.xml create mode 100644 tests/integration/test_refreshable_mat_view/configs/remote_servers.xml create mode 100644 tests/integration/test_refreshable_mat_view/configs/settings.xml create mode 100644 tests/integration/test_refreshable_mat_view/schedule_model.py create mode 100644 tests/integration/test_refreshable_mat_view/test.py create mode 100644 tests/integration/test_refreshable_mat_view/test_schedule_model.py create mode 100644 tests/integration/test_refreshable_mat_view_db_replicated/__init__.py create mode 100644 tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index 141538d502c..a6f9f394871 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -157,13 +157,14 @@ For your convenience, the old documentation is located [here](https://pastila.nl ## Refreshable Materialized View {#refreshable-materialized-view} ```sql -CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name +CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] REFRESH EVERY|AFTER interval [OFFSET interval] -RANDOMIZE FOR interval -DEPENDS ON [db.]name [, [db.]name [, ...]] -SETTINGS name = value [, name = value [, ...]] +[RANDOMIZE FOR interval] +[DEPENDS ON [db.]name [, [db.]name [, ...]]] +[SETTINGS name = value [, name = value [, ...]]] [APPEND] -[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY] +[TO[db.]name] [(columns)] [ENGINE = engine] +[EMPTY] AS SELECT ... [COMMENT 'comment'] ``` diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 1677a1536c5..634bf1fc701 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -4,6 +4,8 @@ import subprocess as sp import tempfile from threading import Timer +import numpy as np + DEFAULT_QUERY_TIMEOUT = 600 @@ -59,6 +61,7 @@ class Client: host=None, ignore_error=False, query_id=None, + parse=False, ): return self.get_query_request( sql, @@ -71,6 +74,7 @@ class Client: host=host, ignore_error=ignore_error, query_id=query_id, + parse=parse, ).get_answer() def get_query_request( @@ -85,6 +89,7 @@ class Client: host=None, ignore_error=False, query_id=None, + parse=False, ): command = self.command[:] @@ -107,8 +112,10 @@ class Client: command += ["--host", host] if query_id is not None: command += ["--query_id", query_id] + if parse: + command += ["--format=TabSeparatedWithNames"] - return CommandRequest(command, stdin, timeout, ignore_error) + return CommandRequest(command, stdin, timeout, ignore_error, parse) @stacktraces_on_timeout_decorator def query_and_get_error( @@ -169,7 +176,9 @@ class QueryRuntimeException(Exception): class CommandRequest: - def __init__(self, command, stdin=None, timeout=None, ignore_error=False): + def __init__( + self, command, stdin=None, timeout=None, ignore_error=False, parse=False + ): # Write data to tmp file to avoid PIPEs and execution blocking stdin_file = tempfile.TemporaryFile(mode="w+") stdin_file.write(stdin) @@ -177,7 +186,7 @@ class CommandRequest: self.stdout_file = tempfile.TemporaryFile() self.stderr_file = tempfile.TemporaryFile() self.ignore_error = ignore_error - + self.parse = parse # print " ".join(command) # we suppress stderror on client becase sometimes thread sanitizer @@ -243,6 +252,16 @@ class CommandRequest: stderr, ) + if self.parse: + import pandas as pd + from io import StringIO + + return ( + pd.read_csv(StringIO(stdout), sep="\t") + .replace(r"\N", None) + .replace(np.nan, None) + ) + return stdout def get_error(self): diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index d47bd5a1c01..31c10b7d053 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -25,6 +25,7 @@ from typing import Any, List, Sequence, Tuple, Union import requests import urllib3 +import shlex try: # Please, add modules that required for specific tests only here. @@ -49,6 +50,7 @@ except Exception as e: logging.warning(f"Cannot import some modules, some tests may not work: {e}") import docker +from docker.models.containers import Container from dict2xml import dict2xml from kazoo.client import KazooClient from kazoo.exceptions import KazooException @@ -525,7 +527,7 @@ class ClickHouseCluster: self.base_jdbc_bridge_cmd = [] self.base_redis_cmd = [] self.pre_zookeeper_commands = [] - self.instances = {} + self.instances: dict[str, ClickHouseInstance] = {} self.with_zookeeper = False self.with_zookeeper_secure = False self.with_mysql_client = False @@ -765,7 +767,7 @@ class ClickHouseCluster: self.prometheus_remote_read_handler_port = 9092 self.prometheus_remote_read_handler_path = "/read" - self.docker_client = None + self.docker_client: docker.DockerClient = None self.is_up = False self.env = os.environ.copy() logging.debug(f"CLUSTER INIT base_config_dir:{self.base_config_dir}") @@ -956,7 +958,7 @@ class ClickHouseCluster: except: pass - def get_docker_handle(self, docker_id): + def get_docker_handle(self, docker_id) -> Container: exception = None for i in range(20): try: @@ -3353,6 +3355,12 @@ class ClickHouseCluster: logging.info("Starting zookeeper node: %s", n) subprocess_check_call(self.base_zookeeper_cmd + ["start", n]) + def query_all_nodes(self, sql, *args, **kwargs): + return { + name: instance.query(sql, ignore_error=True, *args, **kwargs) + for name, instance in self.instances.items() + } + DOCKER_COMPOSE_TEMPLATE = """--- services: @@ -3628,6 +3636,7 @@ class ClickHouseInstance: host=None, ignore_error=False, query_id=None, + parse=False, ): sql_for_log = "" if len(sql) > 1000: @@ -3646,6 +3655,7 @@ class ClickHouseInstance: ignore_error=ignore_error, query_id=query_id, host=host, + parse=parse, ) def query_with_retry( @@ -3662,6 +3672,7 @@ class ClickHouseInstance: retry_count=20, sleep_time=0.5, check_callback=lambda x: True, + parse=False, ): # logging.debug(f"Executing query {sql} on {self.name}") result = None @@ -3678,6 +3689,7 @@ class ClickHouseInstance: database=database, host=host, ignore_error=ignore_error, + parse=parse, ) if check_callback(result): return result @@ -4410,7 +4422,7 @@ class ClickHouseInstance: else: self.wait_start(time_left) - def get_docker_handle(self): + def get_docker_handle(self) -> Container: return self.cluster.get_docker_handle(self.docker_id) def stop(self): diff --git a/tests/integration/test_refreshable_mat_view/configs/config.xml b/tests/integration/test_refreshable_mat_view/configs/config.xml new file mode 100644 index 00000000000..3cbf717bb67 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/configs/config.xml @@ -0,0 +1,2 @@ + + diff --git a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml new file mode 100644 index 00000000000..6504141033a --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml @@ -0,0 +1,27 @@ + + + + + + node1_1 + 9000 + + + node1_2 + 9000 + + + + + + node2_1 + 9000 + + + node2_2 + 9000 + + + + + diff --git a/tests/integration/test_refreshable_mat_view/configs/settings.xml b/tests/integration/test_refreshable_mat_view/configs/settings.xml new file mode 100644 index 00000000000..d15acfaa303 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/configs/settings.xml @@ -0,0 +1,8 @@ + + + + 1 + Etc/UTC + + + diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py new file mode 100644 index 00000000000..857a1285076 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/schedule_model.py @@ -0,0 +1,143 @@ +from datetime import datetime, timedelta + +from dateutil.relativedelta import relativedelta + +""" +It is the model to test the scheduling of refreshable mat view +""" + + +def relative_offset(unit, value): + if unit == "SECONDS": + return relativedelta(seconds=value) + elif unit == "MINUTE": + return relativedelta(minutes=value) + elif unit == "HOUR": + return relativedelta(hours=value) + elif unit == "DAY": + return relativedelta(days=value) + elif unit == "WEEK": + return relativedelta(days=7 * value) + elif unit == "MONTH": + return relativedelta(months=value) + elif unit == "YEAR": + return relativedelta(years=value) + + raise Exception("Cant parse unit: {}".format(unit)) + + +def group_and_sort(parts, reverse=False): + order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECONDS"] + grouped_parts = [] + + for i in range(0, len(parts), 2): + grouped_parts.append((parts[i], parts[i + 1])) + + sorted_parts = sorted( + grouped_parts, key=lambda x: order.index(x[1]), reverse=reverse + ) + return sorted_parts + + +def get_next_refresh_time(schedule, current_time: datetime): + parts = schedule.split() + + randomize_offset = timedelta() + + if "RANDOMIZE" in parts: + randomize_index = parts.index("RANDOMIZE") + randomize_parts = parts[randomize_index + 2 :] + + for i in range(0, len(randomize_parts), 2): + value = int(randomize_parts[i]) + randomize_offset += relative_offset(randomize_parts[i + 1], value) + + parts = parts[:randomize_index] + + offset = timedelta() + if "OFFSET" in parts: + offset_index = parts.index("OFFSET") + for i in range(offset_index + 1, len(parts), 2): + value = int(parts[i]) + offset += relative_offset(parts[i + 1], value) + + parts = parts[:offset_index] + + if parts[0] == "EVERY": + parts = group_and_sort(parts[1:]) + for part in parts: + value = int(part[0]) + unit = part[1] + + if unit == "SECONDS": + current_time = current_time.replace(microsecond=0) + relativedelta( + seconds=value + ) + elif unit == "MINUTE": + current_time = current_time.replace( + second=0, microsecond=0 + ) + relativedelta(minutes=value) + elif unit == "HOUR": + current_time = current_time.replace( + minute=0, second=0, microsecond=0 + ) + relativedelta(hours=value) + elif unit == "DAY": + current_time = current_time.replace( + hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(days=value) + elif unit == "WEEK": + current_time = current_time.replace( + hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(weekday=0, weeks=value) + elif unit == "MONTH": + current_time = current_time.replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(months=value) + elif unit == "YEAR": + current_time = current_time.replace( + month=1, day=1, hour=0, minute=0, second=0, microsecond=0 + ) + relativedelta(years=value) + + current_time += offset + + if randomize_offset: + half_offset = (current_time + randomize_offset - current_time) / 2 + return ( + current_time - half_offset, + current_time + half_offset, + ) + + return current_time + + elif parts[0] == "AFTER": + parts = group_and_sort(parts[1:], reverse=True) + interval = relativedelta() + for part in parts: + value = int(part[0]) + unit = part[1] + + if unit == "SECONDS": + interval += relativedelta(seconds=value) + elif unit == "MINUTE": + interval += relativedelta(minutes=value) + elif unit == "HOUR": + interval += relativedelta(hours=value) + elif unit == "DAY": + interval += relativedelta(days=value) + elif unit == "WEEK": + interval += relativedelta(weeks=value) + elif unit == "MONTH": + interval += relativedelta(months=value) + elif unit == "YEAR": + interval += relativedelta(years=value) + + current_time += interval + if randomize_offset: + half_offset = (current_time + randomize_offset - current_time) / 2 + return ( + current_time - half_offset, + current_time + half_offset, + ) + + return current_time + raise ValueError("Invalid refresh schedule") diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py new file mode 100644 index 00000000000..48c29072134 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -0,0 +1,2530 @@ +import datetime +import os +import random +import shutil +import time +import re +from typing import Optional + +import numpy as np +import pytest +import threading + +from dateutil.relativedelta import relativedelta +from jinja2 import Template, Environment +from datetime import datetime, timedelta + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain +from helpers.network import PartitionManager +from test_refreshable_mat_view.schedule_model import get_next_refresh_time + +test_recover_staled_replica_run = 1 + +cluster = ClickHouseCluster(__file__) + +node1_1 = cluster.add_instance( + "node1_1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 1, "replica": 1}, +) +node1_2 = cluster.add_instance( + "node1_2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 1, "replica": 2}, +) +node2_1 = cluster.add_instance( + "node2_1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 2, "replica": 1}, +) + +node2_2 = cluster.add_instance( + "node2_2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + macros={"shard": 2, "replica": 2}, +) + +# all_nodes = [ +# main_node, +# dummy_node, +# competing_node, +# ] + +uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +def assert_create_query(nodes, table_name, expected): + replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) + query = "show create table {}".format(table_name) + for node in nodes: + assert_eq_with_retry(node, query, expected, get_result=replace_uuid) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(scope="module", autouse=True) +def setup_tables(started_cluster): + print(node1_1.query("SELECT version()")) + + node1_1.query(f"CREATE DATABASE test_db ON CLUSTER test_cluster ENGINE = Atomic") + + node1_1.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + + node1_1.query( + f"CREATE MATERIALIZED VIEW dummy_rmv ON CLUSTER test_cluster " + f"REFRESH EVERY 10 HOUR engine Memory AS select number as x from numbers(10)" + ) + + +""" +#!/usr/bin/env bash +# Tags: atomic-database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# reset --log_comment +CLICKHOUSE_LOG_COMMENT= +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Set session timezone to UTC to make all DateTime formatting and parsing use UTC, because refresh +# scheduling is done in UTC. +CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" +CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" + +$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" + + +# Basic refreshing. +$CLICKHOUSE_CLIENT -nq " + create materialized view a + refresh after 2 second + engine Memory + empty + as select number as x from numbers(2) union all select rand64() as x; + select '<1: created view>', exception, view from refreshes; + show create a;" +# Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) +while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time is null from refreshes -- $LINENO" | xargs`" != '0' ] +do + sleep 0.1 +done +start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" +# Check table contents. +$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" +# Wait for table contents to change. +res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" +while : +do + res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + [ "$res2" == "$res1" ] || break + sleep 0.1 +done +# Wait for another change. +while : +do + res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + [ "$res3" == "$res2" ] || break + sleep 0.1 +done +# Check that the two changes were at least 1 second apart, in particular that we're not refreshing +# like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer +# to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. +$CLICKHOUSE_CLIENT -nq " + select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); + select '<4: next refresh in>', next_refresh_time-last_success_time from refreshes;" + +# Create a source table from which views will read. +$CLICKHOUSE_CLIENT -nq " + create table src (x Int8) engine Memory as select 1;" + +# Switch to fake clock, change refresh schedule, change query. +$CLICKHOUSE_CLIENT -nq " + system test view a set fake time '2050-01-01 00:00:01'; + system wait view a; + system refresh view a; + system wait view a; + select '<4.1: fake clock>', status, last_success_time, next_refresh_time from refreshes; + alter table a modify refresh every 2 year; + alter table a modify query select x*2 as x from src; + system wait view a; + select '<4.5: altered>', status, last_success_time, next_refresh_time from refreshes; + show create a;" +# Advance time to trigger the refresh. +$CLICKHOUSE_CLIENT -nq " + select '<5: no refresh>', count() from a; + system test view a set fake time '2052-02-03 04:05:06';" +while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time, status from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06 Scheduled' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<6: refreshed>', * from a; + select '<7: refreshed>', status, last_success_time, next_refresh_time from refreshes;" + +# Create a dependent view, refresh it once. +$CLICKHOUSE_CLIENT -nq " + create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; + show create b; + system test view b set fake time '2052-11-11 11:11:11'; + system refresh view b; + system wait view b; + select '<7.5: created dependent>', last_success_time from refreshes where view = 'b';" +# Next refresh shouldn't start until the dependency refreshes. +$CLICKHOUSE_CLIENT -nq " + select '<8: refreshed>', * from b; + select '<9: refreshed>', view, status, next_refresh_time from refreshes; + system test view b set fake time '2054-01-24 23:22:21';" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +do + sleep 0.1 +done + +# Drop the source table, check that refresh fails and doesn't leave a temp table behind. +$CLICKHOUSE_CLIENT -nq " + select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); + drop table src; + system refresh view a;" +$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -nq " + select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" + +# Create the source table again, check that refresh succeeds (in particular that tables are looked +# up by name rather than uuid). +$CLICKHOUSE_CLIENT -nq " + select '<10: creating>', view, status, next_refresh_time from refreshes; + create table src (x Int16) engine Memory as select 2; + system test view a set fake time '2054-01-01 00:00:01';" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] +do + sleep 0.1 +done +# Both tables should've refreshed. +$CLICKHOUSE_CLIENT -nq " + select '<11: chain-refreshed a>', * from a; + select '<12: chain-refreshed b>', * from b; + select '<13: chain-refreshed>', view, status, last_success_time, last_refresh_time, next_refresh_time, exception == '' from refreshes;" + +$CLICKHOUSE_CLIENT -nq " + system test view b set fake time '2061-01-01 00:00:00'; + truncate src; + insert into src values (3); + system test view a set fake time '2060-02-02 02:02:02';" +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<15: chain-refreshed a>', * from a; + select '<16: chain-refreshed b>', * from b; + select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" + +# Get to WaitingForDependencies state and remove the depencency. +$CLICKHOUSE_CLIENT -nq " + system test view b set fake time '2062-03-03 03:03:03'" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + alter table b modify refresh every 2 year" +while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<18: removed dependency>', view, status, last_success_time, last_refresh_time, next_refresh_time from refreshes where view = 'b'; + show create b;" + +# Select from a table that doesn't exist, get an exception. +$CLICKHOUSE_CLIENT -nq " + drop table a; + drop table b; + create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; + drop table src;" +while [ "`$CLICKHOUSE_CLIENT -nq "select exception == '' from refreshes -- $LINENO" | xargs`" != '0' ] +do + sleep 0.1 +done +# Check exception, create src, expect successful refresh. +$CLICKHOUSE_CLIENT -nq " + select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' from refreshes; + create table src (x Int64) engine Memory as select 1; + system refresh view c; + system wait view c;" +# Rename table. +$CLICKHOUSE_CLIENT -nq " + select '<20: unexception>', * from c; + rename table c to d; + select '<21: rename>', * from d; + select '<22: rename>', view, status, last_success_time is null from refreshes;" + +# Do various things during a refresh. +# First make a nonempty view. +$CLICKHOUSE_CLIENT -nq " + drop table d; + truncate src; + insert into src values (1) + create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x as select x + sleepEachRow(1) as x from src settings max_block_size = 1; + system wait view e;" +# Stop refreshes. +$CLICKHOUSE_CLIENT -nq " + select '<23: simple refresh>', * from e; + system stop view e;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] +do + sleep 0.1 +done +# Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure +# we wait for a slow refresh, not a previous fast one.) +$CLICKHOUSE_CLIENT -nq " + insert into src select * from numbers(1000) settings max_block_size=1; + system start view e;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +do + sleep 0.1 +done +# Rename. +$CLICKHOUSE_CLIENT -nq " + rename table e to f; + select '<24: rename during refresh>', * from f; + select '<25: rename during refresh>', view, status from refreshes; + alter table f modify refresh after 10 year settings refresh_retries = 0;" +sleep 2 # make it likely that at least one row was processed +# Cancel. +$CLICKHOUSE_CLIENT -nq " + system cancel view f;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Scheduled' ] +do + sleep 0.1 +done +# Check that another refresh doesn't immediately start after the cancelled one. +sleep 1 +$CLICKHOUSE_CLIENT -nq " + select '<27: cancelled>', view, status, exception from refreshes; + system refresh view f;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +do + sleep 0.1 +done +# Drop. +$CLICKHOUSE_CLIENT -nq " + drop table f; + select '<28: drop during refresh>', view, status from refreshes; + select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" + +# Try OFFSET and RANDOMIZE FOR. +$CLICKHOUSE_CLIENT -nq " + create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; + show create g; + system test view g set fake time '2050-02-03 15:30:13';" +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + with '2050-02-10 04:00:00'::DateTime as expected + select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" + +# Send data 'TO' an existing table. +$CLICKHOUSE_CLIENT -nq " + drop table g; + create table dest (x Int64) engine MergeTree order by x; + truncate src; + insert into src values (1); + create materialized view h refresh every 1 second to dest as select x*10 as x from src; + show create h; + system wait view h; + select '<30: to existing table>', * from dest; + insert into src values (2);" +while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<31: to existing table>', * from dest; + drop table dest; + drop table h;" + +# Retries. +$CLICKHOUSE_CLIENT -nq " + create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" +$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -nq " + select '<31.5: will retry>', exception != '', retry > 0 from refreshes; + create table src2 empty as src; + insert into src2 values (1) + exchange tables src and src2; + drop table src2;" +while [ "`$CLICKHOUSE_CLIENT -nq "select status, retry from refreshes -- $LINENO" | xargs`" != 'Scheduled 0' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<31.6: did retry>', x from h2; + drop table h2" + +# EMPTY +$CLICKHOUSE_CLIENT -nq " + create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); + system wait view i; + create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" +while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] +do + sleep 0.1 +done +$CLICKHOUSE_CLIENT -nq " + select '<32: empty>', view, status, last_success_time is null, retry from refreshes order by view; + drop table i; + drop table j;" + +# APPEND +$CLICKHOUSE_CLIENT -nq " + create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; + select '<33: append>', * from k; + system refresh view k; + system wait view k; + select '<34: append>', * from k; + truncate table src; + insert into src values (2), (3); + system refresh view k; + system wait view k; + select '<35: append>', * from k order by x;" +# ALTER to non-APPEND +$CLICKHOUSE_CLIENT -nq " + alter table k modify refresh every 10 year;" 2>/dev/null || echo "ALTER from APPEND to non-APPEND failed, as expected" +$CLICKHOUSE_CLIENT -nq " + drop table k; + truncate table src;" + +# APPEND + TO + regular materialized view reading from it. +$CLICKHOUSE_CLIENT -nq " + create table mid (x Int64) engine MergeTree order by x; + create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; + create materialized view m (x Int64) engine Memory as select x*10 as x from mid; + insert into src values (1); + system refresh view l; + system wait view l; + select '<37: append chain>', * from m; + insert into src values (2); + system refresh view l; + system wait view l; + select '<38: append chain>', * from m order by x; + drop table l; + drop table m; + drop table mid;" + +# Failing to create inner table. +$CLICKHOUSE_CLIENT -nq " + create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" +$CLICKHOUSE_CLIENT -nq " + create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); + drop table n;" + +$CLICKHOUSE_CLIENT -nq " + drop table refreshes;" + +Information about Refreshable Materialized Views. Contains all refreshable materialized views, regardless of whether there's a refresh in progress or not. + +Columns: + +database (String) — The name of the database the table is in. +view (String) — Table name. +status (String) — Current state of the refresh. +last_success_time (Nullable(DateTime)) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation. +last_success_duration_ms (Nullable(UInt64)) — How long the latest refresh took. +last_refresh_time (Nullable(DateTime)) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation. +last_refresh_replica (String) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt. +next_refresh_time (Nullable(DateTime)) — Time at which the next refresh is scheduled to start, if status = Scheduled. +exception (String) — Error message from previous attempt if it failed. +retry (UInt64) — How many failed attempts there were so far, for the current refresh. +progress (Float64) — Progress of the current refresh, between 0 and 1. Not available if status is RunningOnAnotherReplica. +read_rows (UInt64) — Number of rows read by the current refresh so far. Not available if status is RunningOnAnotherReplica. +total_rows (UInt64) — Estimated total number of rows that need to be read by the current refresh. Not available if status is RunningOnAnotherReplica. + +Available refresh settings: + * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. + * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. + * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). + + + +CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name +REFRESH EVERY|AFTER interval [OFFSET interval] +RANDOMIZE FOR interval +DEPENDS ON [db.]name [, [db.]name [, ...]] +SETTINGS name = value [, name = value [, ...]] +[APPEND] +[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY] +AS SELECT ... +where interval is a sequence of simple intervals: + +number SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR + +Example refresh schedules: +```sql +REFRESH EVERY 1 DAY -- every day, at midnight (UTC) +REFRESH EVERY 1 MONTH -- on 1st day of every month, at midnight +REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00 am +REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm +REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc +REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day +-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER +``` + +`RANDOMIZE FOR` randomly adjusts the time of each refresh, e.g.: +```sql +REFRESH EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR -- every day at random time between 01:30 and 02:30 + + +SYSTEM REFRESH VIEW + +Wait for the currently running refresh to complete. If the refresh fails, throws an exception. If no refresh is running, completes immediately, throwing an exception if previous refresh failed. + +### SYSTEM STOP VIEW, SYSTEM STOP VIEWS + +### SYSTEM WAIT VIEW + +Waits for the running refresh to complete. If no refresh is running, returns immediately. If the latest refresh attempt failed, reports an error. + +Can be used right after creating a new refreshable materialized view (without EMPTY keyword) to wait for the initial refresh to complete. + +```sql +SYSTEM WAIT VIEW [db.]name + +### TESTS +1. ON CLUSTER? +2. Append mode +3. Drop node and wait for restore +4. Simple functional testing: all values in refresh result correct (two and more rmv) +5. Combinations of time +6. Two (and more) rmv from single to single [APPEND] +7. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] +8. RMV without tgt table (automatic table) (check APPEND) + +8 DROP rmv +9 CREATE - DROP - CREATE - ALTER +10 Backups? +11. Long queries over refresh time (check settings) +12. RMV settings +13. incorrect intervals (interval 1 sec, offset 1 minute) +14. ALTER on cluster + +15. write to distributed with / without APPEND +16. `default_replica_path` empty on database replicated +17. Not existent TO table (ON CLUSTER) +18. Not existent FROM table (ON CLUSTER) +19. Not existent BOTH tables (ON CLUSTER) +20. retry failed +21. overflow with wait test + +22. ALTER of SELECT? + +23. OFFSET must be less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' + + + +""" + + +def sql_template( + string: str, + globals: Optional[dict] = None, + filters: Optional[dict] = None, + tests: Optional[dict] = None, +) -> Template: + def uppercase(value: str): + return value.upper() + + def lowercase(value: str): + return value.lower() + + def format_settings(items: dict): + return ", ".join([f"{k}={v}" for k, v in items.items()]) + + # Create a custom environment and add the functions + env = Environment( + trim_blocks=False, lstrip_blocks=True, keep_trailing_newline=False + ) + env.globals["uppercase"] = uppercase + env.globals["lowercase"] = lowercase + env.filters["format_settings"] = format_settings + + if filters: + env.filters.update(filters) + if globals: + env.globals.update(globals) + if tests: + env.tests.update(tests) + + return env.from_string(string) + + +def assert_same_values(lst: list): + if not isinstance(lst, list): + lst = list(lst) + assert all(x == lst[0] for x in lst) + + +CREATE_RMV_TEMPLATE = sql_template( + """CREATE MATERIALIZED VIEW +{% if if_not_exists %}IF NOT EXISTS{% endif %} +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +REFRESH {{ refresh_interval }} +{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %} +{% if with_append %}APPEND{% endif %} +{% if to_clause %}TO {{ to_clause }}{% endif %} +{% if table_clause %}{{ table_clause }}{% endif %} +{% if empty %}EMPTY{% endif %} +AS {{ select_query }}""" +) + +# ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] + +ALTER_RMV_TEMPLATE = sql_template( + """ALTER TABLE +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +MODIFY REFRESH {{ refresh_interval }} +{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %}""" +) + + +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("create_target_table", [True, False]) +@pytest.mark.parametrize("if_not_exists", [True, False]) +@pytest.mark.parametrize("on_cluster", [True, False]) +@pytest.mark.parametrize( + "depends_on", [None, ["dummy_rmv"], ["default.dummy_rmv", "src1"]] +) +@pytest.mark.parametrize("randomize", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) +def test_correct_states( + request, + started_cluster, + with_append, + create_target_table, + if_not_exists, + on_cluster, + depends_on, + randomize, + empty, + database_name, +): + """ + Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... + Check ref + """ + + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + + request.addfinalizer(teardown) + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db_name=database_name, + refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + depends_on=depends_on, + to_clause="tgt1", + select_query="SELECT * FROM src1", + with_append=with_append, + on_cluster="test_cluster" if on_cluster else None, + # settings={'setting1':'value1', 'setting2': 'value2'}, + ) + print(create_sql) + node1_1.query(create_sql) + refreshes = node1_1.query( + "SELECT hostname(), * FROM clusterAllReplicas('test_cluster', system.view_refreshes)", + parse=True, + ) + + # Check same RMV is created on cluster + def compare_create_all_nodes(): + show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") + + if not on_cluster: + del show_create_all_nodes["node1_1"] + + assert_same_values(show_create_all_nodes.values()) + + compare_create_all_nodes() + + show_create = node1_1.query("SHOW CREATE test_rmv") + + alter_sql = ALTER_RMV_TEMPLATE.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db_name=database_name, + refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + depends_on=depends_on, + select_query="SELECT * FROM src1", + on_cluster="test_cluster" if on_cluster else None, + # settings={'setting1':'value1', 'setting2': 'value2'}, + ) + + node1_1.query(alter_sql) + show_create_after = node1_1.query("SHOW CREATE test_rmv") + + compare_create_all_nodes() + assert show_create == show_create_after + # breakpoint() + + pass + + +def compare_dates( + date1: str | datetime, + date2: str | datetime, + inaccuracy=timedelta(minutes=10), + format_str="%Y-%m-%d %H:%M:%S", +) -> bool: + """ + Compares two dates with an inaccuracy of 2 minutes. + """ + if isinstance(date1, str): + date1 = datetime.strptime(date1, format_str) + if isinstance(date2, str): + date2 = datetime.strptime(date2, format_str) + + return abs(date1 - date2) <= inaccuracy + + +def date_in_interval( + date1: str | datetime, + date2: str | datetime, + inaccuracy=timedelta(minutes=2), + format_str="%Y-%m-%d %H:%M:%S", +): + pass + + +def get_rmv_info(node, table): + return node.query_with_retry( + f"SELECT * FROM system.view_refreshes WHERE view='{table}'", + check_callback=lambda r: r.iloc[0]["status"] == "Scheduled", + parse=True, + ).to_dict("records")[0] + + +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("create_target_table", [True, False]) +# @pytest.mark.parametrize("if_not_exists", [True, False]) +@pytest.mark.parametrize("on_cluster", [True, False]) +@pytest.mark.parametrize("depends_on", [None, ["dummy_rmv"]]) +@pytest.mark.parametrize("randomize", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) +def test_check_data( + request, + started_cluster, + with_append, + create_target_table, + # if_not_exists, + on_cluster, + depends_on, + randomize, + empty, + database_name, +): + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") + node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") + + request.addfinalizer(teardown) + + # CREATE RMV can use already existed table + # or create new one + if create_target_table: + to_clause = "tgt1" + tgt = "tgt1" + else: + to_clause = "tgt_new Engine = Memory" + tgt = "tgt_new" + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + # if_not_exists=if_not_exists, + db_name=database_name, + refresh_interval="EVERY 1 HOUR", + randomize_interval="30 MINUTE" if randomize else None, + depends_on=depends_on, + to_clause=to_clause, + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=with_append, + on_cluster="test_cluster" if on_cluster else None, + empty=empty + # settings={'setting1':'value1', 'setting2': 'value2'}, + ) + print(create_sql) + node1_1.query(create_sql) + + # now = node1_1.query("SELECT now()", parse=True)[0][0] + now = datetime.utcnow() + + rmv = get_rmv_info(node1_1, "test_rmv") + print(rmv) + + """ + 1. check table view_refreshes + 2. check inserted data if without EMPTY + 3. set time, wait for refresh + 4. check data inserted + 5. alter table + """ + + assert rmv["exception"] is None + assert rmv["status"] == "Scheduled" + assert rmv["progress"] is None + + if not empty: + # Insert + assert compare_dates( + rmv["last_refresh_time"], now, format_str="%Y-%m-%d %H:%M:%S" + ) + assert rmv["last_success_duration_ms"] > 0 + inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) + print(inserted_data) + assert len(inserted_data) == 2 + + if empty: + assert rmv["last_refresh_time"] is None + assert rmv["last_success_duration_ms"] is None + + assert rmv["retry"] == 0 + assert rmv["read_rows"] == 0 + assert rmv["read_bytes"] == 0 + assert rmv["total_rows"] == 0 + assert rmv["total_bytes"] == 0 + assert rmv["written_rows"] == 0 + assert rmv["written_bytes"] == 0 + assert rmv["result_rows"] == 0 + assert rmv["result_bytes"] == 0 + + # Rewind to the next trigger and wait for it + # node1_1.query("SYSTEM TEST VIEW test_rmv set fake time '2050-01-01 00:00:01';") + # system test view a set fake time '2050-01-01 00:00:01'; + + now = datetime.utcnow() + + inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) + if with_append: + assert len(inserted_data) == 4 + else: + assert len(inserted_data) == 2 + + # alter time + + breakpoint() + + +""" + +(Pdb) pp interval +'EVERY 1 WEEK' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 6, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 6, 0, 0) + + +(Pdb) pp interval +'EVERY 2 WEEK' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 6, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 13, 0, 0) + + +(Pdb) pp interval +'EVERY 2 WEEK OFFSET 1 DAY' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 7, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 14, 0, 0) + + +(Pdb) pp interval +'EVERY 2 WEEK OFFSET 2 DAY' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 8, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 15, 0, 0) + + +'EVERY 1 WEEK OFFSET 2 DAY' +(Pdb) pp next_refresh_time +datetime.datetime(2024, 5, 1, 0, 0) +(Pdb) pp predicted_next_refresh_time +datetime.datetime(2024, 5, 8, 0, 0) + +""" + + +def parse_ch_datetime(date_str): + return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") + + +INTERVALS = [ + # Same units + # "EVERY 1 YEAR 2 YEAR", + # "AFTER 1 YEAR 2 YEAR" + "EVERY 1 MINUTE", + "EVERY 1 HOUR", + "EVERY 1 DAY", + "EVERY 1 MONTH", + "EVERY 1 YEAR", + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECONDS", + # smth wrong with + # "EVERY 1 WEEK", + # "EVERY 2 WEEK", + # "EVERY 3 WEEK", + # "EVERY 4 WEEK", + # "EVERY 5 WEEK", + # "EVERY 6 WEEK", + # "EVERY 7 WEEK", + # "EVERY 8 WEEK", + # "EVERY 9 WEEK", + # "EVERY 10 WEEK", + # "EVERY 11 WEEK", + # "EVERY 12 WEEK", + # "EVERY 13 WEEK", + # "EVERY 14 WEEK", + # "EVERY 15 WEEK", + # "EVERY 20 WEEK", + # "EVERY 21 WEEK", + # "EVERY 31 WEEK", + # "EVERY 32 WEEK", + # "EVERY 33 WEEK", + # "EVERY 50 WEEK", + # "EVERY 51 WEEK", + # "EVERY 52 WEEK", + # "EVERY 59 WEEK", + # + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", + "EVERY 1 MONTH OFFSET 1 WEEK", + "EVERY 1 MONTH OFFSET 2 WEEK", + # + "AFTER 30 SECONDS", + "AFTER 30 MINUTE", + "AFTER 2 HOUR", + "AFTER 2 DAY", + "AFTER 2 WEEK", + "AFTER 2 MONTH", + "AFTER 2 YEAR", + "AFTER 2 MONTH", + "AFTER 2 MONTH 3 DAY", + "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", + "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", + # Randomize bigger than interval + "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE OFFSET 1 SECONDS RANDOMIZE FOR 3 YEAR", + # + "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", + "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR", + "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", + # + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", + # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS OFFSET 11 MONTH RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", + # + # Two different units in EVERY + # "EVERY 1 YEAR", + # "EVERY 1 YEAR 0 MONTH", + # "EVERY 1 YEAR 1 MONTH", + # "EVERY 1 YEAR 2 MONTH", + # "EVERY 1 YEAR 3 MONTH", + # "EVERY 1 YEAR 4 MONTH", + # "EVERY 1 YEAR 8 MONTH", + # "EVERY 1 YEAR 9 MONTH", + # "EVERY 1 YEAR 10 MONTH", + # "EVERY 1 YEAR 11 MONTH", + # "EVERY 1 YEAR 12 MONTH", + # "EVERY 1 YEAR 13 MONTH", + # "EVERY 1 DAY", + # "EVERY 1 DAY 0 HOUR", + # "EVERY 1 DAY 1 HOUR", + # "EVERY 1 DAY 2 HOUR", + # "EVERY 1 DAY 28 HOUR", + # "EVERY 1 DAY 29 HOUR", + # "EVERY 1 DAY 30 HOUR", + # "EVERY 1 DAY 31 HOUR", + # "EVERY 1 DAY 32 HOUR", + # + "AFTER 1 MONTH", + "AFTER 1 MONTH 0 DAY", + "AFTER 1 MONTH 1 DAY", + "AFTER 1 MONTH 3 DAY", + "AFTER 1 MONTH 50 DAY", + "AFTER 1 YEAR 10 MONTH", + "AFTER 1 YEAR 10 MONTH 3 DAY", + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR", + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", + # Interval shouldn't contain both calendar units and clock units (e.g. months and days) + # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", +] + + +@pytest.mark.parametrize( + "interval", + INTERVALS, +) +@pytest.mark.parametrize( + "append", + [ + # True, + False + ], +) +@pytest.mark.parametrize( + "empty", + [ + True, + # False + ], +) +def test_schedule( + request, + started_cluster, + interval, + append, + empty, +): + """ + - Create RMV + - Check table view_refreshes + - Check inserted data if without EMPTY + - Set time, wait for refresh + - Check data is inserted/appended + - Alter table + - Check everything again + - DROP target table + """ + + if "WEEK" in interval: + pytest.skip() + + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") + # node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") + # node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") + + request.addfinalizer(teardown) + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv_schedule", + refresh_interval=interval, + table_clause="ENGINE = Memory", + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=append, + empty=empty, + ) + print(create_sql) + node1_1.query(create_sql) + + now = datetime.utcnow() + + rmv = get_rmv_info(node1_1, "test_rmv_schedule") + print(rmv) + + next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) + predicted_next_refresh_time = get_next_refresh_time(interval, now) + + print("----") + print("current_time:", now) + print("Interval:", interval) + print("next_refresh_time", next_refresh_time) + print("predicted", predicted_next_refresh_time) + print("----") + + # RANDOMIZE means the next refresh time will be randomly chosen + # within a range of RANDOMIZE interval + if "RANDOMIZE" in interval: + assert ( + predicted_next_refresh_time[0] + <= next_refresh_time + <= predicted_next_refresh_time[1] + ) + else: + assert compare_dates(next_refresh_time, predicted_next_refresh_time) + + assert next_refresh_time > now + + append_expect_rows = 0 + + def check_data(): + inserted_data = node1_1.query_with_retry( + f"SELECT * FROM test_rmv_schedule", + parse=True, + check_callback=lambda x: len(x) > 0, + retry_count=200, + ) + + # Check data if not EMPTY + if append: + # Append adds rows + global append_expect_rows + append_expect_rows += 2 + assert len(inserted_data) == append_expect_rows + if not append: + # Rewrite without append + assert len(inserted_data) == 2 + + inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) + if empty: + assert len(inserted_data) == 0 + else: + assert rmv["last_success_duration_ms"] > 0 + check_data() + + # Trigger next refresh + node1_1.query( + f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" + ) + + if "RANDOMIZE" not in interval: + check_data() + + # rmv = get_rmv_info(node1_1, "test_rmv_schedule") + # next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) + + # # Alter RMV to random interval and test schedule is changed + # # TODO: remove week filter after fix + # interval_alter = random.choice(list(filter(lambda x: "WEEK" not in x, INTERVALS))) + # alter_sql = ALTER_RMV_TEMPLATE.render( + # table_name="test_rmv_schedule", + # refresh_interval=interval_alter, + # # select_query="SELECT * FROM src1", + # ) + # print(alter_sql) + # node1_1.query(alter_sql) + # now_alter = datetime.utcnow() + # + # rmv = get_rmv_info(node1_1, "test_rmv_schedule") + # next_refresh_time_alter = parse_ch_datetime(rmv["next_refresh_time"]) + # predicted_next_refresh_time = get_next_refresh_time(interval_alter, now_alter) + # + # if "RANDOMIZE" in interval_alter: + # assert ( + # predicted_next_refresh_time[0] + # <= next_refresh_time_alter + # <= predicted_next_refresh_time[1] + # ) + # else: + # assert compare_dates(next_refresh_time_alter, predicted_next_refresh_time) + # + # assert next_refresh_time_alter > now_alter + # + # # Trigger next refresh + # node1_1.query( + # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time_alter}'" + # ) + # check_data() + + # breakpoint() + + +# def test_create_replicated_table(started_cluster): +# main_node.query( +# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica' || '1');" +# ) +# dummy_node.query( +# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica2');" +# ) +# assert ( +# "Explicit zookeeper_path and replica_name are specified" +# in main_node.query_and_get_error( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " +# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# ) +# +# assert ( +# "Explicit zookeeper_path and replica_name are specified" +# in main_node.query_and_get_error( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " +# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# ) +# +# assert ( +# "This syntax for *MergeTree engine is deprecated" +# in main_node.query_and_get_error( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " +# "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" +# ) +# ) +# +# main_node.query( +# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# +# expected = ( +# "CREATE TABLE create_replicated_table.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\n" +# "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" +# ) +# assert_create_query( +# [main_node, dummy_node], "create_replicated_table.replicated_table", expected +# ) +# # assert without replacing uuid +# assert main_node.query( +# "show create create_replicated_table.replicated_table" +# ) == dummy_node.query("show create create_replicated_table.replicated_table") +# main_node.query("DROP DATABASE create_replicated_table SYNC") +# dummy_node.query("DROP DATABASE create_replicated_table SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_simple_alter_table(started_cluster, engine): +# database = f"test_simple_alter_table_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# # test_simple_alter_table +# name = f"{database}.alter_test" +# main_node.query( +# "CREATE TABLE {} " +# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " +# "ENGINE = {} PARTITION BY StartDate ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);".format( +# name, engine +# ) +# ) +# main_node.query("ALTER TABLE {} ADD COLUMN Added0 UInt32;".format(name)) +# main_node.query("ALTER TABLE {} ADD COLUMN Added2 UInt32;".format(name)) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN Added1 UInt32 AFTER Added0;".format(name) +# ) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;".format( +# name +# ) +# ) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;".format( +# name +# ) +# ) +# main_node.query( +# "ALTER TABLE {} ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;".format( +# name +# ) +# ) +# +# full_engine = ( +# engine +# if not "Replicated" in engine +# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" +# ) +# expected = ( +# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" +# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n" +# " `AddedNested1.A` Array(UInt32),\\n `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n" +# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64)\\n)\\n" +# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" +# "SETTINGS index_granularity = 8192".format(name, full_engine) +# ) +# +# assert_create_query([main_node, dummy_node], name, expected) +# +# # test_create_replica_after_delay +# competing_node.query( +# f"CREATE DATABASE IF NOT EXISTS {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica3');" +# ) +# +# main_node.query("ALTER TABLE {} ADD COLUMN Added3 UInt32;".format(name)) +# main_node.query("ALTER TABLE {} DROP COLUMN AddedNested1;".format(name)) +# main_node.query("ALTER TABLE {} RENAME COLUMN Added1 TO AddedNested1;".format(name)) +# +# full_engine = ( +# engine +# if not "Replicated" in engine +# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" +# ) +# expected = ( +# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" +# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `AddedNested1` UInt32,\\n `Added2` UInt32,\\n" +# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64),\\n `Added3` UInt32\\n)\\n" +# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" +# "SETTINGS index_granularity = 8192".format(name, full_engine) +# ) +# +# assert_create_query([main_node, dummy_node, competing_node], name, expected) +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# competing_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_delete_from_table(started_cluster, engine): +# database = f"delete_from_table_{engine}" +# +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" +# ) +# +# name = f"{database}.delete_test" +# main_node.query( +# "CREATE TABLE {} " +# "(id UInt64, value String) " +# "ENGINE = {} PARTITION BY id%2 ORDER BY (id);".format(name, engine) +# ) +# main_node.query("INSERT INTO TABLE {} VALUES(1, 'aaaa');".format(name)) +# main_node.query("INSERT INTO TABLE {} VALUES(2, 'aaaa');".format(name)) +# dummy_node.query("INSERT INTO TABLE {} VALUES(1, 'bbbb');".format(name)) +# dummy_node.query("INSERT INTO TABLE {} VALUES(2, 'bbbb');".format(name)) +# +# main_node.query("DELETE FROM {} WHERE id=2;".format(name)) +# +# expected = "1\taaaa\n1\tbbbb" +# +# table_for_select = name +# if not "Replicated" in engine: +# table_for_select = f"cluster('{database}', {name})" +# for node in [main_node, dummy_node]: +# assert_eq_with_retry( +# node, +# "SELECT * FROM {} ORDER BY id, value;".format(table_for_select), +# expected, +# ) +# +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# def get_table_uuid(database, name): +# return main_node.query( +# f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'" +# ).strip() +# +# +# @pytest.fixture(scope="module", name="attachable_part") +# def fixture_attachable_part(started_cluster): +# main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") +# main_node.query( +# f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO testdb_attach_atomic.test VALUES (123)") +# main_node.query( +# f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'" +# ) +# table_uuid = get_table_uuid("testdb_attach_atomic", "test") +# return os.path.join( +# main_node.path, +# f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0", +# ) +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_attach(started_cluster, attachable_part, engine): +# database = f"alter_attach_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# main_node.query( +# f"CREATE TABLE {database}.alter_attach_test (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# table_uuid = get_table_uuid(database, "alter_attach_test") +# # Provide and attach a part to the main node +# shutil.copytree( +# attachable_part, +# os.path.join( +# main_node.path, +# f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0", +# ), +# ) +# main_node.query(f"ALTER TABLE {database}.alter_attach_test ATTACH PART 'all_1_1_0'") +# # On the main node, data is attached +# assert ( +# main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") +# == "123\n" +# ) +# # On the other node, data is replicated only if using a Replicated table engine +# if engine == "ReplicatedMergeTree": +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") +# == "123\n" +# ) +# else: +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") +# == "" +# ) +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_drop_part(started_cluster, engine): +# database = f"alter_drop_part_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" +# main_node.query( +# f"CREATE TABLE {database}.alter_drop_part (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (123)") +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (456)") +# else: +# main_node.query(f"SYSTEM SYNC REPLICA {database}.alter_drop_part PULL") +# main_node.query(f"ALTER TABLE {database}.alter_drop_part DROP PART '{part_name}'") +# assert main_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" +# if engine == "ReplicatedMergeTree": +# # The DROP operation is still replicated at the table engine level +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" +# ) +# else: +# assert ( +# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") +# == "456\n" +# ) +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_detach_part(started_cluster, engine): +# database = f"alter_detach_part_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" +# main_node.query( +# f"CREATE TABLE {database}.alter_detach (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_detach VALUES (123)") +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_detach VALUES (456)") +# main_node.query(f"ALTER TABLE {database}.alter_detach DETACH PART '{part_name}'") +# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_detach'" +# assert main_node.query(detached_parts_query) == f"{part_name}\n" +# if engine == "ReplicatedMergeTree": +# # The detach operation is still replicated at the table engine level +# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" +# else: +# assert dummy_node.query(detached_parts_query) == "" +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_drop_detached_part(started_cluster, engine): +# database = f"alter_drop_detached_part_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# +# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" +# main_node.query( +# f"CREATE TABLE {database}.alter_drop_detached (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (123)") +# main_node.query( +# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" +# ) +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (456)") +# dummy_node.query( +# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" +# ) +# main_node.query( +# f"ALTER TABLE {database}.alter_drop_detached DROP DETACHED PART '{part_name}'" +# ) +# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_drop_detached'" +# assert main_node.query(detached_parts_query) == "" +# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" +# +# main_node.query(f"DROP DATABASE {database} SYNC") +# dummy_node.query(f"DROP DATABASE {database} SYNC") +# +# +# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) +# def test_alter_drop_partition(started_cluster, engine): +# database = f"alter_drop_partition_{engine}" +# main_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" +# ) +# snapshotting_node.query( +# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" +# ) +# +# main_node.query( +# f"CREATE TABLE {database}.alter_drop (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" +# ) +# main_node.query(f"INSERT INTO {database}.alter_drop VALUES (123)") +# if engine == "MergeTree": +# dummy_node.query(f"INSERT INTO {database}.alter_drop VALUES (456)") +# snapshotting_node.query(f"INSERT INTO {database}.alter_drop VALUES (789)") +# main_node.query( +# f"ALTER TABLE {database}.alter_drop ON CLUSTER {database} DROP PARTITION ID 'all'", +# settings={"replication_alter_partitions_sync": 2}, +# ) +# assert ( +# main_node.query( +# f"SELECT CounterID FROM clusterAllReplicas('{database}', {database}.alter_drop)" +# ) +# == "" +# ) +# assert dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop") == "" +# main_node.query(f"DROP DATABASE {database}") +# dummy_node.query(f"DROP DATABASE {database}") +# snapshotting_node.query(f"DROP DATABASE {database}") +# +# +# def test_alter_fetch(started_cluster): +# main_node.query( +# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica2');" +# ) +# +# main_node.query( +# "CREATE TABLE alter_fetch.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" +# ) +# main_node.query( +# "CREATE TABLE alter_fetch.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" +# ) +# main_node.query("INSERT INTO alter_fetch.fetch_source VALUES (123)") +# table_uuid = get_table_uuid("alter_fetch", "fetch_source") +# main_node.query( +# f"ALTER TABLE alter_fetch.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " +# ) +# detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='alter_fetch' AND table='fetch_target'" +# assert main_node.query(detached_parts_query) == "all_0_0_0\n" +# assert dummy_node.query(detached_parts_query) == "" +# +# main_node.query("DROP DATABASE alter_fetch SYNC") +# dummy_node.query("DROP DATABASE alter_fetch SYNC") +# +# +# def test_alters_from_different_replicas(started_cluster): +# main_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica2');" +# ) +# +# # test_alters_from_different_replicas +# competing_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica3');" +# ) +# +# main_node.query( +# "CREATE TABLE alters_from_different_replicas.concurrent_test " +# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " +# "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" +# ) +# +# main_node.query( +# "CREATE TABLE alters_from_different_replicas.dist AS alters_from_different_replicas.concurrent_test ENGINE = Distributed(alters_from_different_replicas, alters_from_different_replicas, concurrent_test, CounterID)" +# ) +# +# dummy_node.stop_clickhouse(kill=True) +# +# settings = {"distributed_ddl_task_timeout": 5} +# assert "is not finished on 1 of 3 hosts" in competing_node.query_and_get_error( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;", +# settings=settings, +# ) +# settings = { +# "distributed_ddl_task_timeout": 5, +# "distributed_ddl_output_mode": "null_status_on_timeout", +# } +# assert "shard1\treplica2\tQUEUED\t" in main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added2 UInt32;", +# settings=settings, +# ) +# settings = { +# "distributed_ddl_task_timeout": 5, +# "distributed_ddl_output_mode": "never_throw", +# } +# assert "shard1\treplica2\tQUEUED\t" in competing_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", +# settings=settings, +# ) +# dummy_node.start_clickhouse() +# main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" +# ) +# competing_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" +# ) +# main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" +# ) +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" +# " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" +# " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" +# " `AddedNested2.B` Array(UInt64)\\n)\\n" +# "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" +# ) +# +# assert_create_query( +# [main_node, competing_node], +# "alters_from_different_replicas.concurrent_test", +# expected, +# ) +# +# # test_create_replica_after_delay +# main_node.query("DROP TABLE alters_from_different_replicas.concurrent_test SYNC") +# main_node.query( +# "CREATE TABLE alters_from_different_replicas.concurrent_test " +# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " +# "ENGINE = ReplicatedMergeTree ORDER BY CounterID;" +# ) +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" +# ) +# +# assert_create_query( +# [main_node, competing_node], +# "alters_from_different_replicas.concurrent_test", +# expected, +# ) +# +# main_node.query( +# "INSERT INTO alters_from_different_replicas.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" +# ) +# +# # test_replica_restart +# main_node.restart_clickhouse() +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" +# ) +# +# # test_snapshot_and_snapshot_recover +# snapshotting_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica1');" +# ) +# snapshot_recovering_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica2');" +# ) +# assert_create_query( +# all_nodes, "alters_from_different_replicas.concurrent_test", expected +# ) +# +# main_node.query("SYSTEM FLUSH DISTRIBUTED alters_from_different_replicas.dist") +# main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" +# ) +# res = main_node.query( +# "ALTER TABLE alters_from_different_replicas.concurrent_test DELETE WHERE UserID % 2" +# ) +# assert ( +# "shard1\treplica1\tOK" in res +# and "shard1\treplica2\tOK" in res +# and "shard1\treplica3\tOK" in res +# ) +# assert "shard2\treplica1\tOK" in res and "shard2\treplica2\tOK" in res +# +# expected = ( +# "1\t1\tmain_node\n" +# "1\t2\tdummy_node\n" +# "1\t3\tcompeting_node\n" +# "2\t1\tsnapshotting_node\n" +# "2\t2\tsnapshot_recovering_node\n" +# ) +# assert ( +# main_node.query( +# "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='alters_from_different_replicas'" +# ) +# == expected +# ) +# +# # test_drop_and_create_replica +# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# main_node.query( +# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" +# ) +# +# expected = ( +# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" +# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" +# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" +# ) +# +# assert_create_query( +# [main_node, competing_node], +# "alters_from_different_replicas.concurrent_test", +# expected, +# ) +# assert_create_query( +# all_nodes, "alters_from_different_replicas.concurrent_test", expected +# ) +# +# for node in all_nodes: +# node.query("SYSTEM SYNC REPLICA alters_from_different_replicas.concurrent_test") +# +# expected = ( +# "0\t2021-02-02\t4249604106\n" +# "1\t2021-02-03\t1343103100\n" +# "4\t2021-02-06\t3902320246\n" +# "7\t2021-02-09\t3844986530\n" +# "9\t2021-02-11\t1241149650\n" +# ) +# +# assert_eq_with_retry( +# dummy_node, +# "SELECT CounterID, StartDate, UserID FROM alters_from_different_replicas.dist ORDER BY CounterID", +# expected, +# ) +# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# dummy_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# competing_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# snapshotting_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# snapshot_recovering_node.query("DROP DATABASE alters_from_different_replicas SYNC") +# +# +# def create_some_tables(db): +# settings = { +# "distributed_ddl_task_timeout": 0, +# "allow_experimental_object_type": 1, +# "allow_suspicious_codecs": 1, +# } +# main_node.query(f"CREATE TABLE {db}.t1 (n int) ENGINE=Memory", settings=settings) +# dummy_node.query( +# f"CREATE TABLE {db}.t2 (s String) ENGINE=Memory", settings=settings +# ) +# main_node.query( +# f"CREATE TABLE {db}.mt1 (n int) ENGINE=MergeTree order by n", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE TABLE {db}.mt2 (n int) ENGINE=MergeTree order by n", +# settings=settings, +# ) +# main_node.query( +# f"CREATE TABLE {db}.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE TABLE {db}.rmt2 (n int CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12))) ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# main_node.query( +# f"CREATE TABLE {db}.rmt3 (n int, json Object('json') materialized '') ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE TABLE {db}.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", +# settings=settings, +# ) +# main_node.query( +# f"CREATE MATERIALIZED VIEW {db}.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", +# settings=settings, +# ) +# dummy_node.query( +# f"CREATE MATERIALIZED VIEW {db}.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", +# settings=settings, +# ) +# main_node.query( +# f"CREATE DICTIONARY {db}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " +# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " +# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" +# ) +# dummy_node.query( +# f"CREATE DICTIONARY {db}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " +# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " +# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" +# ) +# +# +# # These tables are used to check that DatabaseReplicated correctly renames all the tables in case when it restores from the lost state +# def create_table_for_exchanges(db): +# settings = {"distributed_ddl_task_timeout": 0} +# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: +# main_node.query( +# f"CREATE TABLE {db}.{table} (s String) ENGINE=ReplicatedMergeTree order by s", +# settings=settings, +# ) +# +# +# def test_recover_staled_replica(started_cluster): +# main_node.query( +# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');" +# ) +# started_cluster.get_kazoo_client("zoo1").set( +# "/clickhouse/databases/recover/logs_to_keep", b"10" +# ) +# dummy_node.query( +# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica2');" +# ) +# +# settings = {"distributed_ddl_task_timeout": 0} +# create_some_tables("recover") +# create_table_for_exchanges("recover") +# +# for table in ["t1", "t2", "mt1", "mt2", "rmt1", "rmt2", "rmt3", "rmt5"]: +# main_node.query(f"INSERT INTO recover.{table} VALUES (42)") +# for table in ["t1", "t2", "mt1", "mt2"]: +# dummy_node.query(f"INSERT INTO recover.{table} VALUES (42)") +# +# for i, table in enumerate(["a1", "a2", "a3", "a4", "a5", "a6"]): +# main_node.query(f"INSERT INTO recover.{table} VALUES ('{str(i + 1) * 10}')") +# +# for table in ["rmt1", "rmt2", "rmt3", "rmt5"]: +# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") +# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: +# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") +# +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(dummy_node) +# dummy_node.query_and_get_error("RENAME TABLE recover.t1 TO recover.m1") +# +# main_node.query_with_retry( +# "RENAME TABLE recover.t1 TO recover.m1", settings=settings +# ) +# main_node.query_with_retry( +# "ALTER TABLE recover.mt1 ADD COLUMN m int", settings=settings +# ) +# main_node.query_with_retry( +# "ALTER TABLE recover.rmt1 ADD COLUMN m int", settings=settings +# ) +# main_node.query_with_retry( +# "RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings +# ) +# main_node.query_with_retry("DROP TABLE recover.rmt5", settings=settings) +# main_node.query_with_retry("DROP DICTIONARY recover.d2", settings=settings) +# main_node.query_with_retry( +# "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " +# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " +# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", +# settings=settings, +# ) +# +# inner_table = ( +# ".inner_id." +# + dummy_node.query_with_retry( +# "SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'" +# ).strip() +# ) +# main_node.query_with_retry( +# f"ALTER TABLE recover.`{inner_table}` MODIFY COLUMN n int DEFAULT 42", +# settings=settings, +# ) +# main_node.query_with_retry( +# "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m as n FROM recover.rmt1", +# settings=settings, +# ) +# main_node.query_with_retry( +# "RENAME TABLE recover.mv2 TO recover.mv3", +# settings=settings, +# ) +# +# main_node.query_with_retry( +# "CREATE TABLE recover.tmp AS recover.m1", settings=settings +# ) +# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) +# main_node.query_with_retry( +# "CREATE TABLE recover.tmp AS recover.m1", settings=settings +# ) +# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) +# main_node.query_with_retry( +# "CREATE TABLE recover.tmp AS recover.m1", settings=settings +# ) +# +# main_node.query("EXCHANGE TABLES recover.a1 AND recover.a2", settings=settings) +# main_node.query("EXCHANGE TABLES recover.a3 AND recover.a4", settings=settings) +# main_node.query("EXCHANGE TABLES recover.a5 AND recover.a4", settings=settings) +# main_node.query("EXCHANGE TABLES recover.a6 AND recover.a3", settings=settings) +# main_node.query("RENAME TABLE recover.a6 TO recover.a7", settings=settings) +# main_node.query("RENAME TABLE recover.a1 TO recover.a8", settings=settings) +# +# assert ( +# main_node.query( +# "SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name" +# ) +# == "a2\na3\na4\na5\na7\na8\nd1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" +# ) +# query = ( +# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " +# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" +# ) +# expected = main_node.query(query) +# assert_eq_with_retry(dummy_node, query, expected) +# assert ( +# main_node.query( +# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" +# ) +# == "2\n" +# ) +# assert ( +# dummy_node.query( +# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" +# ) +# == "2\n" +# ) +# +# # Check that Database Replicated renamed all the tables correctly +# for i, table in enumerate(["a2", "a8", "a5", "a7", "a4", "a3"]): +# assert ( +# dummy_node.query(f"SELECT * FROM recover.{table}") == f"{str(i + 1) * 10}\n" +# ) +# +# for table in [ +# "m1", +# "t2", +# "mt1", +# "mt2", +# "rmt1", +# "rmt2", +# "rmt4", +# "d1", +# "d2", +# "mv1", +# "mv3", +# ]: +# assert main_node.query(f"SELECT (*,).1 FROM recover.{table}") == "42\n" +# for table in ["t2", "rmt1", "rmt2", "rmt4", "d1", "d2", "mt2", "mv1", "mv3"]: +# assert ( +# dummy_node.query(f"SELECT '{table}', (*,).1 FROM recover.{table}") +# == f"{table}\t42\n" +# ) +# for table in ["m1", "mt1"]: +# assert dummy_node.query(f"SELECT count() FROM recover.{table}") == "0\n" +# global test_recover_staled_replica_run +# assert ( +# dummy_node.query( +# "SELECT count() FROM system.tables WHERE database='recover_broken_tables'" +# ) +# == f"{test_recover_staled_replica_run}\n" +# ) +# assert ( +# dummy_node.query( +# "SELECT count() FROM system.tables WHERE database='recover_broken_replicated_tables'" +# ) +# == f"{test_recover_staled_replica_run}\n" +# ) +# test_recover_staled_replica_run += 1 +# +# print(dummy_node.query("SHOW DATABASES")) +# print(dummy_node.query("SHOW TABLES FROM recover_broken_tables")) +# print(dummy_node.query("SHOW TABLES FROM recover_broken_replicated_tables")) +# +# table = dummy_node.query( +# "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_41_%' LIMIT 1" +# ).strip() +# assert ( +# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_tables.{table}") == "42\n" +# ) +# table = dummy_node.query( +# "SHOW TABLES FROM recover_broken_replicated_tables LIKE 'rmt5_41_%' LIMIT 1" +# ).strip() +# assert ( +# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_replicated_tables.{table}") +# == "42\n" +# ) +# +# expected = "Cleaned 6 outdated objects: dropped 1 dictionaries and 3 tables, moved 2 tables" +# assert_logs_contain(dummy_node, expected) +# +# dummy_node.query("DROP TABLE recover.tmp") +# assert_eq_with_retry( +# main_node, +# "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", +# "0\n", +# ) +# main_node.query("DROP DATABASE recover SYNC") +# dummy_node.query("DROP DATABASE recover SYNC") +# +# +# def test_recover_staled_replica_many_mvs(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS recover_mvs") +# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") +# +# main_node.query_with_retry( +# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica1');" +# ) +# started_cluster.get_kazoo_client("zoo1").set( +# "/clickhouse/databases/recover_mvs/logs_to_keep", b"10" +# ) +# dummy_node.query_with_retry( +# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica2');" +# ) +# +# settings = {"distributed_ddl_task_timeout": 0} +# +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(dummy_node) +# dummy_node.query_and_get_error("RENAME TABLE recover_mvs.t1 TO recover_mvs.m1") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query( +# f"CREATE TABLE recover_mvs.rmt{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", +# settings=settings, +# ) +# +# print("Created tables") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query( +# f"CREATE TABLE recover_mvs.mv_inner{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", +# settings=settings, +# ) +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE MATERIALIZED VIEW recover_mvs.mv{identifier} +# TO recover_mvs.mv_inner{identifier} +# AS SELECT * FROM recover_mvs.rmt{identifier}""", +# settings=settings, +# ) +# +# print("Created MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.view_from_mv{identifier} +# AS SELECT * FROM recover_mvs.mv{identifier}""", +# settings=settings, +# ) +# +# print("Created Views on top of MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE MATERIALIZED VIEW recover_mvs.cascade_mv{identifier} +# ENGINE=MergeTree() ORDER BY tuple() +# POPULATE AS SELECT * FROM recover_mvs.mv_inner{identifier};""", +# settings=settings, +# ) +# +# print("Created cascade MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.view_from_cascade_mv{identifier} +# AS SELECT * FROM recover_mvs.cascade_mv{identifier}""", +# settings=settings, +# ) +# +# print("Created Views on top of cascade MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE MATERIALIZED VIEW recover_mvs.double_cascade_mv{identifier} +# ENGINE=MergeTree() ORDER BY tuple() +# POPULATE AS SELECT * FROM recover_mvs.`.inner_id.{get_table_uuid("recover_mvs", f"cascade_mv{identifier}")}`""", +# settings=settings, +# ) +# +# print("Created double cascade MVs") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.view_from_double_cascade_mv{identifier} +# AS SELECT * FROM recover_mvs.double_cascade_mv{identifier}""", +# settings=settings, +# ) +# +# print("Created Views on top of double cascade MVs") +# +# # This weird table name is actually makes sence because it starts with letter `a` and may break some internal sorting +# main_node.query_with_retry( +# """ +# CREATE VIEW recover_mvs.anime +# AS +# SELECT n +# FROM +# ( +# SELECT * +# FROM +# ( +# SELECT * +# FROM +# ( +# SELECT * +# FROM recover_mvs.mv_inner1 AS q1 +# INNER JOIN recover_mvs.mv_inner2 AS q2 ON q1.n = q2.n +# ) AS new_table_1 +# INNER JOIN recover_mvs.mv_inner3 AS q3 ON new_table_1.n = q3.n +# ) AS new_table_2 +# INNER JOIN recover_mvs.mv_inner4 AS q4 ON new_table_2.n = q4.n +# ) +# """, +# settings=settings, +# ) +# +# print("Created final boss") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE DICTIONARY recover_mvs.`11111d{identifier}` (n UInt64) +# PRIMARY KEY n +# SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'double_cascade_mv{identifier}' DB 'recover_mvs')) +# LAYOUT(FLAT()) LIFETIME(1)""", +# settings=settings, +# ) +# +# print("Created dictionaries") +# +# for identifier in ["1", "2", "3", "4"]: +# main_node.query_with_retry( +# f"""CREATE VIEW recover_mvs.`00000vd{identifier}` +# AS SELECT * FROM recover_mvs.`11111d{identifier}`""", +# settings=settings, +# ) +# +# print("Created Views on top of dictionaries") +# +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_mvs") +# query = "SELECT name FROM system.tables WHERE database='recover_mvs' ORDER BY name" +# assert main_node.query(query) == dummy_node.query(query) +# +# main_node.query("DROP DATABASE IF EXISTS recover_mvs") +# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") +# +# +# def test_startup_without_zk(started_cluster): +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(main_node) +# err = main_node.query_and_get_error( +# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" +# ) +# assert "ZooKeeper" in err or "Coordination::Exception" in err +# main_node.query( +# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" +# ) +# main_node.query( +# "CREATE TABLE startup.rmt (n int) ENGINE=ReplicatedMergeTree order by n" +# ) +# +# main_node.query("INSERT INTO startup.rmt VALUES (42)") +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(main_node) +# main_node.restart_clickhouse(stop_start_wait_sec=60) +# assert main_node.query("SELECT (*,).1 FROM startup.rmt") == "42\n" +# +# # we need to wait until the table is not readonly +# main_node.query_with_retry("INSERT INTO startup.rmt VALUES(42)") +# +# main_node.query_with_retry("CREATE TABLE startup.m (n int) ENGINE=Memory") +# +# main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") +# assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" +# +# main_node.query("DROP DATABASE startup SYNC") +# +# +# def test_server_uuid(started_cluster): +# uuid1 = main_node.query("select serverUUID()") +# uuid2 = dummy_node.query("select serverUUID()") +# assert uuid1 != uuid2 +# main_node.restart_clickhouse() +# uuid1_after_restart = main_node.query("select serverUUID()") +# assert uuid1 == uuid1_after_restart +# +# +# def test_sync_replica(started_cluster): +# main_node.query( +# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica2');" +# ) +# +# number_of_tables = 1000 +# +# settings = {"distributed_ddl_task_timeout": 0} +# +# with PartitionManager() as pm: +# pm.drop_instance_zk_connections(dummy_node) +# +# for i in range(number_of_tables): +# main_node.query( +# "CREATE TABLE test_sync_database.table_{} (n int) ENGINE=MergeTree order by n".format( +# i +# ), +# settings=settings, +# ) +# +# # wait for host to reconnect +# dummy_node.query_with_retry("SELECT * FROM system.zookeeper WHERE path='/'") +# +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") +# +# assert "2\n" == main_node.query( +# "SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'" +# ) +# +# assert dummy_node.query( +# "SELECT count() FROM system.tables where database='test_sync_database'" +# ).strip() == str(number_of_tables) +# +# assert main_node.query( +# "SELECT count() FROM system.tables where database='test_sync_database'" +# ).strip() == str(number_of_tables) +# +# engine_settings = {"default_table_engine": "ReplicatedMergeTree"} +# dummy_node.query( +# "CREATE TABLE test_sync_database.table (n int, primary key n) partition by n", +# settings=engine_settings, +# ) +# main_node.query("INSERT INTO test_sync_database.table SELECT * FROM numbers(10)") +# dummy_node.query("TRUNCATE TABLE test_sync_database.table", settings=settings) +# dummy_node.query( +# "ALTER TABLE test_sync_database.table ADD COLUMN m int", settings=settings +# ) +# +# main_node.query( +# "SYSTEM SYNC DATABASE REPLICA ON CLUSTER test_sync_database test_sync_database" +# ) +# +# lp1 = main_node.query( +# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica1' and name='log_ptr'" +# ) +# lp2 = main_node.query( +# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica2' and name='log_ptr'" +# ) +# max_lp = main_node.query( +# "select value from system.zookeeper where path='/test/sync_replica/' and name='max_log_ptr'" +# ) +# assert lp1 == max_lp +# assert lp2 == max_lp +# +# main_node.query("DROP DATABASE test_sync_database SYNC") +# dummy_node.query("DROP DATABASE test_sync_database SYNC") +# +# +# def test_force_synchronous_settings(started_cluster): +# main_node.query( +# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica2');" +# ) +# snapshotting_node.query( +# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard2', 'replica1');" +# ) +# main_node.query( +# "CREATE TABLE test_force_synchronous_settings.t (n int) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" +# ) +# main_node.query( +# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" +# ) +# snapshotting_node.query( +# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" +# ) +# snapshotting_node.query( +# "SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings" +# ) +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings") +# +# snapshotting_node.query("SYSTEM STOP MERGES test_force_synchronous_settings.t") +# +# def start_merges_func(): +# time.sleep(5) +# snapshotting_node.query("SYSTEM START MERGES test_force_synchronous_settings.t") +# +# start_merges_thread = threading.Thread(target=start_merges_func) +# start_merges_thread.start() +# +# settings = { +# "mutations_sync": 2, +# "database_replicated_enforce_synchronous_settings": 1, +# } +# main_node.query( +# "ALTER TABLE test_force_synchronous_settings.t UPDATE n = n * 10 WHERE 1", +# settings=settings, +# ) +# assert "10\t450\n" == snapshotting_node.query( +# "SELECT count(), sum(n) FROM test_force_synchronous_settings.t" +# ) +# start_merges_thread.join() +# +# def select_func(): +# dummy_node.query( +# "SELECT sleepEachRow(1) FROM test_force_synchronous_settings.t SETTINGS function_sleep_max_microseconds_per_block = 0" +# ) +# +# select_thread = threading.Thread(target=select_func) +# select_thread.start() +# +# settings = {"database_replicated_enforce_synchronous_settings": 1} +# snapshotting_node.query( +# "DROP TABLE test_force_synchronous_settings.t SYNC", settings=settings +# ) +# main_node.query( +# "CREATE TABLE test_force_synchronous_settings.t (n String) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" +# ) +# select_thread.join() +# +# +# def test_recover_digest_mismatch(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# +# main_node.query( +# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica2');" +# ) +# +# create_some_tables("recover_digest_mismatch") +# +# main_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") +# +# ways_to_corrupt_metadata = [ +# "mv /var/lib/clickhouse/metadata/recover_digest_mismatch/t1.sql /var/lib/clickhouse/metadata/recover_digest_mismatch/m1.sql", +# "sed --follow-symlinks -i 's/Int32/String/' /var/lib/clickhouse/metadata/recover_digest_mismatch/mv1.sql", +# "rm -f /var/lib/clickhouse/metadata/recover_digest_mismatch/d1.sql", +# "rm -rf /var/lib/clickhouse/metadata/recover_digest_mismatch/", # Will trigger "Directory already exists" +# "rm -rf /var/lib/clickhouse/store", +# ] +# +# for command in ways_to_corrupt_metadata: +# print(f"Corrupting data using `{command}`") +# need_remove_is_active_node = "rm -rf" in command +# dummy_node.stop_clickhouse(kill=not need_remove_is_active_node) +# dummy_node.exec_in_container(["bash", "-c", command]) +# +# query = ( +# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover_digest_mismatch' AND name NOT LIKE '.inner_id.%' " +# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" +# ) +# expected = main_node.query(query) +# +# if need_remove_is_active_node: +# # NOTE Otherwise it fails to recreate ReplicatedMergeTree table due to "Replica already exists" +# main_node.query( +# "SYSTEM DROP REPLICA '2' FROM DATABASE recover_digest_mismatch" +# ) +# +# # There is a race condition between deleting active node and creating it on server startup +# # So we start a server only after we deleted all table replicas from the Keeper +# dummy_node.start_clickhouse() +# assert_eq_with_retry(dummy_node, query, expected) +# +# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") +# +# print("Everything Okay") +# +# +# def test_replicated_table_structure_alter(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS table_structure") +# dummy_node.query("DROP DATABASE IF EXISTS table_structure") +# +# main_node.query( +# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica2');" +# ) +# competing_node.query( +# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica3');" +# ) +# +# competing_node.query("CREATE TABLE table_structure.mem (n int) ENGINE=Memory") +# dummy_node.query("DETACH DATABASE table_structure") +# +# settings = {"distributed_ddl_task_timeout": 0} +# main_node.query( +# "CREATE TABLE table_structure.rmt (n int, v UInt64) ENGINE=ReplicatedReplacingMergeTree(v) ORDER BY n", +# settings=settings, +# ) +# +# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# competing_node.query("DETACH DATABASE table_structure") +# +# main_node.query( +# "ALTER TABLE table_structure.rmt ADD COLUMN m int", settings=settings +# ) +# main_node.query( +# "ALTER TABLE table_structure.rmt COMMENT COLUMN v 'version'", settings=settings +# ) +# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3)") +# +# command = "rm -f /var/lib/clickhouse/metadata/table_structure/mem.sql" +# competing_node.exec_in_container(["bash", "-c", command]) +# competing_node.restart_clickhouse(kill=True) +# +# dummy_node.query("ATTACH DATABASE table_structure") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") +# assert "1\t2\t3\n" == dummy_node.query("SELECT * FROM table_structure.rmt") +# +# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# competing_node.query("SYSTEM SYNC REPLICA table_structure.rmt") +# # time.sleep(600) +# assert "mem" in competing_node.query("SHOW TABLES FROM table_structure") +# assert "1\t2\t3\n" == competing_node.query("SELECT * FROM table_structure.rmt") +# +# main_node.query("ALTER TABLE table_structure.rmt ADD COLUMN k int") +# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3, 4)") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") +# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") +# assert "1\t2\t3\t0\n1\t2\t3\t4\n" == dummy_node.query( +# "SELECT * FROM table_structure.rmt ORDER BY k" +# ) +# +# +# def test_modify_comment(started_cluster): +# main_node.query( +# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '1');" +# ) +# +# dummy_node.query( +# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '2');" +# ) +# +# main_node.query( +# "CREATE TABLE modify_comment_db.modify_comment_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" +# ) +# +# def restart_verify_not_readonly(): +# main_node.restart_clickhouse() +# assert ( +# main_node.query( +# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" +# ) +# == "0\n" +# ) +# dummy_node.restart_clickhouse() +# assert ( +# dummy_node.query( +# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" +# ) +# == "0\n" +# ) +# +# main_node.query( +# "ALTER TABLE modify_comment_db.modify_comment_table COMMENT COLUMN d 'Some comment'" +# ) +# +# restart_verify_not_readonly() +# +# main_node.query( +# "ALTER TABLE modify_comment_db.modify_comment_table MODIFY COMMENT 'Some error comment'" +# ) +# +# restart_verify_not_readonly() +# +# main_node.query("DROP DATABASE modify_comment_db SYNC") +# dummy_node.query("DROP DATABASE modify_comment_db SYNC") +# +# +# def test_table_metadata_corruption(started_cluster): +# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") +# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") +# +# main_node.query( +# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica1');" +# ) +# dummy_node.query( +# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica2');" +# ) +# +# create_some_tables("table_metadata_corruption") +# +# main_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") +# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") +# +# # Server should handle this by throwing an exception during table loading, which should lead to server shutdown +# corrupt = "sed --follow-symlinks -i 's/ReplicatedMergeTree/CorruptedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" +# +# print(f"Corrupting metadata using `{corrupt}`") +# dummy_node.stop_clickhouse(kill=True) +# dummy_node.exec_in_container(["bash", "-c", corrupt]) +# +# query = ( +# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='table_metadata_corruption' AND name NOT LIKE '.inner_id.%' " +# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" +# ) +# expected = main_node.query(query) +# +# # We expect clickhouse server to shutdown without LOGICAL_ERRORs or deadlocks +# dummy_node.start_clickhouse(expected_to_fail=True) +# assert not dummy_node.contains_in_log("LOGICAL_ERROR") +# +# fix_corrupt = "sed --follow-symlinks -i 's/CorruptedMergeTree/ReplicatedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" +# print(f"Fix corrupted metadata using `{fix_corrupt}`") +# dummy_node.exec_in_container(["bash", "-c", fix_corrupt]) +# +# dummy_node.start_clickhouse() +# assert_eq_with_retry(dummy_node, query, expected) +# +# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") +# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py new file mode 100644 index 00000000000..ac15333cd11 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view/test_schedule_model.py @@ -0,0 +1,90 @@ +from datetime import datetime + +from test_refreshable_mat_view.schedule_model import get_next_refresh_time + + +def test_refresh_schedules(): + time_ = datetime(2000, 1, 1, 1, 1, 1) + + assert get_next_refresh_time( + "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", time_ + ) == datetime(2000, 1, 1, 1, 1, 1) + + assert get_next_refresh_time("EVERY 1 SECOND", time_) == datetime( + 2000, 1, 1, 1, 1, 2 + ) + assert get_next_refresh_time("EVERY 1 MINUTE", time_) == datetime( + 2000, + 1, + 1, + 1, + 2, + ) + assert get_next_refresh_time("EVERY 1 HOUR", time_) == datetime( + 2000, + 1, + 1, + 2, + ) + assert get_next_refresh_time("EVERY 1 DAY", time_) == datetime(2000, 1, 2) + assert get_next_refresh_time("EVERY 1 WEEK", time_) == datetime(2000, 1, 10) + assert get_next_refresh_time("EVERY 2 WEEK", time_) == datetime(2000, 1, 17) + assert get_next_refresh_time("EVERY 1 MONTH", time_) == datetime(2000, 2, 1) + assert get_next_refresh_time("EVERY 1 YEAR", time_) == datetime(2001, 1, 1) + + assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", time_) == datetime( + 2003, 5, 11 + ) + + # OFFSET + assert get_next_refresh_time( + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ + ) == datetime(2000, 2, 6, 2, 30, 15) + assert get_next_refresh_time( + "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ + ) == datetime(2001, 3, 6, 2, 30, 15) + + assert get_next_refresh_time( + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", time_ + ) == datetime(2000, 1, 22, 15, 10) + + # AFTER + assert get_next_refresh_time("AFTER 30 SECOND", time_) == datetime( + 2000, 1, 1, 1, 1, 31 + ) + assert get_next_refresh_time("AFTER 30 MINUTE", time_) == datetime( + 2000, 1, 1, 1, 31, 1 + ) + assert get_next_refresh_time("AFTER 2 HOUR", time_) == datetime(2000, 1, 1, 3, 1, 1) + assert get_next_refresh_time("AFTER 2 DAY", time_) == datetime(2000, 1, 3, 1, 1, 1) + assert get_next_refresh_time("AFTER 2 WEEK", time_) == datetime( + 2000, 1, 15, 1, 1, 1 + ) + assert get_next_refresh_time("AFTER 2 MONTH", time_) == datetime( + 2000, 3, 1, 1, 1, 1 + ) + assert get_next_refresh_time("AFTER 2 YEAR", time_) == datetime(2002, 1, 1, 1, 1, 1) + + assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", time_) == datetime( + 2002, 2, 1, 1, 1, 1 + ) + + assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", time_) == datetime( + 2002, 2, 1, 1, 1, 1 + ) + + # RANDOMIZE + next_refresh = get_next_refresh_time( + "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", time_ + ) + + assert next_refresh == (datetime(2000, 1, 2, 2, 0), datetime(2000, 1, 2, 3, 0)) + + next_refresh = get_next_refresh_time( + "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", + time_, + ) + assert next_refresh == ( + datetime(2000, 3, 4, 11, 0, 20), + datetime(2000, 3, 7, 12, 0, 20), + ) diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/__init__.py b/tests/integration/test_refreshable_mat_view_db_replicated/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml b/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml new file mode 100644 index 00000000000..7c0e60a044e --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml @@ -0,0 +1,16 @@ + + + + 1 + 1 + 1 + 0 + 0 + + + + + default + + + From 92a9199c7635e89d02b607a3ac69d9e18bb9b870 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Thu, 1 Aug 2024 11:57:36 +0200 Subject: [PATCH 25/38] Tests: add RMV tests --- .../configs/config.xml | 1 + .../configs/remote_servers.xml | 11 - .../schedule_model.py | 66 +- .../test_refreshable_mat_view/test.py | 1721 ++--------------- .../test_schedule_model.py | 133 +- 5 files changed, 312 insertions(+), 1620 deletions(-) diff --git a/tests/integration/test_refreshable_mat_view/configs/config.xml b/tests/integration/test_refreshable_mat_view/configs/config.xml index 3cbf717bb67..fdaad39175f 100644 --- a/tests/integration/test_refreshable_mat_view/configs/config.xml +++ b/tests/integration/test_refreshable_mat_view/configs/config.xml @@ -1,2 +1,3 @@ + Etc/UTC diff --git a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml index 6504141033a..cb35cec266d 100644 --- a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml +++ b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml @@ -10,17 +10,6 @@ node1_2 9000 - - - - - node2_1 - 9000 - - - node2_2 - 9000 - diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py index 857a1285076..5195df3dec6 100644 --- a/tests/integration/test_refreshable_mat_view/schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/schedule_model.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta -from dateutil.relativedelta import relativedelta +import dateutil.relativedelta as rd """ It is the model to test the scheduling of refreshable mat view @@ -8,26 +8,29 @@ It is the model to test the scheduling of refreshable mat view def relative_offset(unit, value): - if unit == "SECONDS": - return relativedelta(seconds=value) + if unit == "SECOND": + return rd.relativedelta(seconds=value) elif unit == "MINUTE": - return relativedelta(minutes=value) + return rd.relativedelta(minutes=value) elif unit == "HOUR": - return relativedelta(hours=value) + return rd.relativedelta(hours=value) elif unit == "DAY": - return relativedelta(days=value) + return rd.relativedelta(days=value) + # elif unit == "WEEK": + # return rd.relativedelta(days=7 * value) elif unit == "WEEK": - return relativedelta(days=7 * value) - elif unit == "MONTH": - return relativedelta(months=value) - elif unit == "YEAR": - return relativedelta(years=value) + return rd.relativedelta(weeks=7 * value) - raise Exception("Cant parse unit: {}".format(unit)) + elif unit == "MONTH": + return rd.relativedelta(months=value) + elif unit == "YEAR": + return rd.relativedelta(years=value) + + raise Exception("Can't parse unit: {}".format(unit)) def group_and_sort(parts, reverse=False): - order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECONDS"] + order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND"] grouped_parts = [] for i in range(0, len(parts), 2): @@ -41,7 +44,6 @@ def group_and_sort(parts, reverse=False): def get_next_refresh_time(schedule, current_time: datetime): parts = schedule.split() - randomize_offset = timedelta() if "RANDOMIZE" in parts: @@ -69,37 +71,36 @@ def get_next_refresh_time(schedule, current_time: datetime): value = int(part[0]) unit = part[1] - if unit == "SECONDS": - current_time = current_time.replace(microsecond=0) + relativedelta( + if unit == "SECOND": + current_time = current_time.replace(microsecond=0) + rd.relativedelta( seconds=value ) elif unit == "MINUTE": current_time = current_time.replace( second=0, microsecond=0 - ) + relativedelta(minutes=value) + ) + rd.relativedelta(minutes=value) elif unit == "HOUR": current_time = current_time.replace( minute=0, second=0, microsecond=0 - ) + relativedelta(hours=value) + ) + rd.relativedelta(hours=value) elif unit == "DAY": current_time = current_time.replace( hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(days=value) + ) + rd.relativedelta(days=value) elif unit == "WEEK": current_time = current_time.replace( hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(weekday=0, weeks=value) + ) + rd.relativedelta(weekday=0, weeks=value) elif unit == "MONTH": current_time = current_time.replace( day=1, hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(months=value) + ) + rd.relativedelta(months=value) elif unit == "YEAR": current_time = current_time.replace( month=1, day=1, hour=0, minute=0, second=0, microsecond=0 - ) + relativedelta(years=value) + ) + rd.relativedelta(years=value) current_time += offset - if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 return ( @@ -111,31 +112,32 @@ def get_next_refresh_time(schedule, current_time: datetime): elif parts[0] == "AFTER": parts = group_and_sort(parts[1:], reverse=True) - interval = relativedelta() + interval = rd.relativedelta() for part in parts: value = int(part[0]) unit = part[1] - if unit == "SECONDS": - interval += relativedelta(seconds=value) + if unit == "SECOND": + interval += rd.relativedelta(seconds=value) elif unit == "MINUTE": - interval += relativedelta(minutes=value) + interval += rd.relativedelta(minutes=value) elif unit == "HOUR": - interval += relativedelta(hours=value) + interval += rd.relativedelta(hours=value) elif unit == "DAY": - interval += relativedelta(days=value) + interval += rd.relativedelta(days=value) elif unit == "WEEK": - interval += relativedelta(weeks=value) + interval += rd.relativedelta(weeks=value) elif unit == "MONTH": - interval += relativedelta(months=value) + interval += rd.relativedelta(months=value) elif unit == "YEAR": - interval += relativedelta(years=value) + interval += rd.relativedelta(years=value) current_time += interval if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 return ( current_time - half_offset, + # current_time, current_time + half_offset, ) diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index 48c29072134..95d15fe369d 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -19,8 +19,6 @@ from helpers.test_tools import assert_eq_with_retry, assert_logs_contain from helpers.network import PartitionManager from test_refreshable_mat_view.schedule_model import get_next_refresh_time -test_recover_staled_replica_run = 1 - cluster = ClickHouseCluster(__file__) node1_1 = cluster.add_instance( @@ -39,36 +37,13 @@ node1_2 = cluster.add_instance( stay_alive=True, macros={"shard": 1, "replica": 2}, ) -node2_1 = cluster.add_instance( - "node2_1", - main_configs=["configs/remote_servers.xml"], - user_configs=["configs/settings.xml"], - with_zookeeper=True, - stay_alive=True, - macros={"shard": 2, "replica": 1}, -) - -node2_2 = cluster.add_instance( - "node2_2", - main_configs=["configs/remote_servers.xml"], - user_configs=["configs/settings.xml"], - with_zookeeper=True, - stay_alive=True, - macros={"shard": 2, "replica": 2}, -) - -# all_nodes = [ -# main_node, -# dummy_node, -# competing_node, -# ] uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") def assert_create_query(nodes, table_name, expected): replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) - query = "show create table {}".format(table_name) + query = "SHOW CREATE TABLE {}".format(table_name) for node in nodes: assert_eq_with_retry(node, query, expected, get_result=replace_uuid) @@ -466,7 +441,7 @@ database (String) — The name of the database the table is in. view (String) — Table name. status (String) — Current state of the refresh. last_success_time (Nullable(DateTime)) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation. -last_success_duration_ms (Nullable(UInt64)) — How long the latest refresh took. +last_success_time (Nullable(UInt64)) — How long the latest refresh took. last_refresh_time (Nullable(DateTime)) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation. last_refresh_replica (String) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt. next_refresh_time (Nullable(DateTime)) — Time at which the next refresh is scheduled to start, if status = Scheduled. @@ -623,6 +598,7 @@ ALTER_RMV_TEMPLATE = sql_template( {% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} MODIFY REFRESH {{ refresh_interval }} {% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if with_append %}APPEND{% endif %} {% if settings %}SETTINGS {{ settings|format_settings }}{% endif %}""" ) @@ -634,7 +610,6 @@ MODIFY REFRESH {{ refresh_interval }} @pytest.mark.parametrize( "depends_on", [None, ["dummy_rmv"], ["default.dummy_rmv", "src1"]] ) -@pytest.mark.parametrize("randomize", [True, False]) @pytest.mark.parametrize("empty", [True, False]) @pytest.mark.parametrize("database_name", [None, "default", "test_db"]) def test_correct_states( @@ -645,7 +620,6 @@ def test_correct_states( if_not_exists, on_cluster, depends_on, - randomize, empty, database_name, ): @@ -663,7 +637,7 @@ def test_correct_states( table_name="test_rmv", if_not_exists=if_not_exists, db_name=database_name, - refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + refresh_interval="EVERY 1 HOUR", depends_on=depends_on, to_clause="tgt1", select_query="SELECT * FROM src1", @@ -691,36 +665,39 @@ def test_correct_states( show_create = node1_1.query("SHOW CREATE test_rmv") + # Alter of RMV replaces all non-specified alter_sql = ALTER_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, db_name=database_name, - refresh_interval="EVERY 1 HOUR" + " 30 MINUTE" if randomize else None, + refresh_interval="EVERY 1 HOUR", depends_on=depends_on, select_query="SELECT * FROM src1", + with_append=with_append, on_cluster="test_cluster" if on_cluster else None, # settings={'setting1':'value1', 'setting2': 'value2'}, ) node1_1.query(alter_sql) - show_create_after = node1_1.query("SHOW CREATE test_rmv") + show_create_after_alter = node1_1.query("SHOW CREATE test_rmv") compare_create_all_nodes() - assert show_create == show_create_after + assert show_create == show_create_after_alter # breakpoint() - pass - def compare_dates( date1: str | datetime, - date2: str | datetime, - inaccuracy=timedelta(minutes=10), + date2: str | datetime | tuple[datetime], + inaccuracy=timedelta(hours=1), format_str="%Y-%m-%d %H:%M:%S", ) -> bool: """ Compares two dates with an inaccuracy of 2 minutes. """ + if isinstance(date2, tuple): + return date2[0] <= date1 <= date2[1] + if isinstance(date1, str): date1 = datetime.strptime(date1, format_str) if isinstance(date2, str): @@ -739,12 +716,17 @@ def date_in_interval( def get_rmv_info(node, table): - return node.query_with_retry( + rmv_info = node.query_with_retry( f"SELECT * FROM system.view_refreshes WHERE view='{table}'", check_callback=lambda r: r.iloc[0]["status"] == "Scheduled", parse=True, ).to_dict("records")[0] + rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) + + return rmv_info + + @pytest.mark.parametrize("with_append", [True, False]) @pytest.mark.parametrize("create_target_table", [True, False]) @@ -822,14 +804,14 @@ def test_check_data( assert compare_dates( rmv["last_refresh_time"], now, format_str="%Y-%m-%d %H:%M:%S" ) - assert rmv["last_success_duration_ms"] > 0 + assert rmv["last_success_time"] > 0 inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) print(inserted_data) assert len(inserted_data) == 2 if empty: assert rmv["last_refresh_time"] is None - assert rmv["last_success_duration_ms"] is None + assert rmv["last_success_time"] is None assert rmv["retry"] == 0 assert rmv["read_rows"] == 0 @@ -842,11 +824,11 @@ def test_check_data( assert rmv["result_bytes"] == 0 # Rewind to the next trigger and wait for it - # node1_1.query("SYSTEM TEST VIEW test_rmv set fake time '2050-01-01 00:00:01';") - # system test view a set fake time '2050-01-01 00:00:01'; + node1_1.query( + f"SYSTEM TEST VIEW test_rmv set fake time '{rmv['next_refresh_time']}';" + ) now = datetime.utcnow() - inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) if with_append: assert len(inserted_data) == 4 @@ -855,8 +837,6 @@ def test_check_data( # alter time - breakpoint() - """ @@ -905,69 +885,29 @@ def parse_ch_datetime(date_str): return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") -INTERVALS = [ +INTERVALS_EVERY = [ # Same units - # "EVERY 1 YEAR 2 YEAR", - # "AFTER 1 YEAR 2 YEAR" + "EVERY 1 YEAR 2 YEAR", + # "EVERY 1 MINUTE", "EVERY 1 HOUR", "EVERY 1 DAY", + "EVERY 1 WEEK", + "EVERY 2 WEEK", "EVERY 1 MONTH", "EVERY 1 YEAR", - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECONDS", + "EVERY 1 WEEK RANDOMIZE FOR 1 HOUR", + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", + "EVERY 1 MONTH OFFSET 1 DAY RANDOMIZE FOR 10 HOUR", # smth wrong with # "EVERY 1 WEEK", # "EVERY 2 WEEK", # "EVERY 3 WEEK", # "EVERY 4 WEEK", # "EVERY 5 WEEK", - # "EVERY 6 WEEK", - # "EVERY 7 WEEK", - # "EVERY 8 WEEK", - # "EVERY 9 WEEK", - # "EVERY 10 WEEK", - # "EVERY 11 WEEK", - # "EVERY 12 WEEK", - # "EVERY 13 WEEK", - # "EVERY 14 WEEK", - # "EVERY 15 WEEK", - # "EVERY 20 WEEK", - # "EVERY 21 WEEK", - # "EVERY 31 WEEK", - # "EVERY 32 WEEK", - # "EVERY 33 WEEK", - # "EVERY 50 WEEK", - # "EVERY 51 WEEK", - # "EVERY 52 WEEK", - # "EVERY 59 WEEK", - # - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", - "EVERY 1 MONTH OFFSET 1 WEEK", - "EVERY 1 MONTH OFFSET 2 WEEK", - # - "AFTER 30 SECONDS", - "AFTER 30 MINUTE", - "AFTER 2 HOUR", - "AFTER 2 DAY", - "AFTER 2 WEEK", - "AFTER 2 MONTH", - "AFTER 2 YEAR", - "AFTER 2 MONTH", - "AFTER 2 MONTH 3 DAY", - "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", - "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", - # Randomize bigger than interval - "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE OFFSET 1 SECONDS RANDOMIZE FOR 3 YEAR", - # - "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", - "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR", - "EVERY 1 YEAR OFFSET 11 MONTH RANDOMIZE FOR 1 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", - # - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", - # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS OFFSET 11 MONTH RANDOMIZE FOR 2 YEAR 10 MONTH 1 DAY 1 HOUR 2 MINUTE 2 SECONDS", - # + # "EVERY 1 MONTH OFFSET 1 WEEK", + # "EVERY 1 MONTH OFFSET 2 WEEK", # Two different units in EVERY # "EVERY 1 YEAR", # "EVERY 1 YEAR 0 MONTH", @@ -990,7 +930,26 @@ INTERVALS = [ # "EVERY 1 DAY 30 HOUR", # "EVERY 1 DAY 31 HOUR", # "EVERY 1 DAY 32 HOUR", - # + # Interval shouldn't contain both calendar units and clock units (e.g. months and days) + # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", +] + +INTERVALS_AFTER = [ + "AFTER 1 YEAR 2 YEAR", + "AFTER 30 SECOND", + "AFTER 30 MINUTE", + "AFTER 2 HOUR", + "AFTER 2 DAY", + "AFTER 2 WEEK", + "AFTER 2 MONTH", + "AFTER 2 YEAR", + "AFTER 2 MONTH 3 DAY", + "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", + "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", + # Randomize bigger than interval + "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", + "EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", "AFTER 1 MONTH", "AFTER 1 MONTH 0 DAY", "AFTER 1 MONTH 1 DAY", @@ -999,15 +958,131 @@ INTERVALS = [ "AFTER 1 YEAR 10 MONTH", "AFTER 1 YEAR 10 MONTH 3 DAY", "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR", - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", - # Interval shouldn't contain both calendar units and clock units (e.g. months and days) - # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECONDS", + "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", ] @pytest.mark.parametrize( "interval", - INTERVALS, + INTERVALS_EVERY, +) +@pytest.mark.parametrize( + "append", + [ + # True, + False + ], +) +@pytest.mark.parametrize( + "empty", + [ + True, + # False + ], +) +def test_schedule_2( + request, + started_cluster, + interval, + append, + empty, +): + """ + - Create RMV + - Check table view_refreshes + - Check inserted data if without EMPTY + - Set time, wait for refresh + - Check data is inserted/appended + - Alter table + - Check everything again + - DROP target table + """ + + # if "AFTER" in interval: + # pytest.skip() + + def teardown(): + node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") + node1_1.query("DROP TABLE IF EXISTS tgt_new") + node1_1.query("TRUNCATE TABLE tgt1") + + request.addfinalizer(teardown) + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv_schedule", + refresh_interval=interval, + table_clause="ENGINE = Memory", + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=append, + empty=empty, + ) + print(create_sql) + node1_1.query(create_sql) + + now = datetime.utcnow() + + rmv = get_rmv_info(node1_1, "test_rmv_schedule") + # this is for EVERY + predicted_next_refresh_time = get_next_refresh_time(interval, now) + # this is for AFTER + # diff = next_refresh_time - now.replace(microsecond=0) + + if 'WEEK' in + assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) + assert rmv["next_refresh_time"] > now + + def expect_rows(rows): + inserted_data = node1_1.query_with_retry( + f"SELECT * FROM test_rmv_schedule", + parse=True, + check_callback=lambda x: len(x) == rows, + retry_count=100, + ) + assert len(inserted_data) == rows + + # Check data if not EMPTY + append_expect_rows = 0 + if append: + # Append adds rows + append_expect_rows += 2 + expect_rows(append_expect_rows) + if not append: + # Rewrite without append + expect_rows(2) + + inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) + if empty: + # No data is inserted with empty + assert len(inserted_data) == 0 + else: + # Query is executed without empty + assert rmv["last_success_time"] > 0 + expect_rows(2) + + # Trigger refresh, update `next_refresh_time` and check interval again + node1_1.query( + f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{rmv['next_refresh_time']}'" + ) + predicted_next_refresh_time = get_next_refresh_time(interval, rmv['next_refresh_time']) + rmv = get_rmv_info(node1_1, "test_rmv_schedule") + + assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) + + # Check data if not EMPTY + if append: + # Append adds rows + append_expect_rows += 2 + expect_rows(append_expect_rows) + else: + # Rewrite + expect_rows(2) + + # breakpoint() + + +@pytest.mark.parametrize( + "interval", + INTERVALS_EVERY, ) @pytest.mark.parametrize( "append", @@ -1041,13 +1116,13 @@ def test_schedule( - DROP target table """ - if "WEEK" in interval: - pytest.skip() + # if "WEEK" in interval: + # pytest.skip() def teardown(): node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") - # node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") - # node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") + node1_1.query("DROP TABLE IF EXISTS tgt_new") + node1_1.query("TRUNCATE TABLE tgt1") request.addfinalizer(teardown) @@ -1090,41 +1165,41 @@ def test_schedule( assert next_refresh_time > now - append_expect_rows = 0 - - def check_data(): - inserted_data = node1_1.query_with_retry( - f"SELECT * FROM test_rmv_schedule", - parse=True, - check_callback=lambda x: len(x) > 0, - retry_count=200, - ) - - # Check data if not EMPTY - if append: - # Append adds rows - global append_expect_rows - append_expect_rows += 2 - assert len(inserted_data) == append_expect_rows - if not append: - # Rewrite without append - assert len(inserted_data) == 2 - - inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) - if empty: - assert len(inserted_data) == 0 - else: - assert rmv["last_success_duration_ms"] > 0 - check_data() - - # Trigger next refresh - node1_1.query( - f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" - ) - - if "RANDOMIZE" not in interval: - check_data() - + # append_expect_rows = 0 + # + # def check_data(): + # inserted_data = node1_1.query_with_retry( + # f"SELECT * FROM test_rmv_schedule", + # parse=True, + # check_callback=lambda x: len(x) > 0, + # retry_count=200, + # ) + # + # # Check data if not EMPTY + # if append: + # # Append adds rows + # global append_expect_rows + # append_expect_rows += 2 + # assert len(inserted_data) == append_expect_rows + # if not append: + # # Rewrite without append + # assert len(inserted_data) == 2 + # + # inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) + # if empty: + # assert len(inserted_data) == 0 + # else: + # assert rmv["last_success_time"] > 0 + # check_data() + # + # # Trigger next refresh + # node1_1.query( + # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" + # ) + # + # if "RANDOMIZE" not in interval: + # check_data() + # ----------------------------- # rmv = get_rmv_info(node1_1, "test_rmv_schedule") # next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) @@ -1162,1369 +1237,3 @@ def test_schedule( # check_data() # breakpoint() - - -# def test_create_replicated_table(started_cluster): -# main_node.query( -# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica' || '1');" -# ) -# dummy_node.query( -# "CREATE DATABASE create_replicated_table ENGINE = Replicated('/test/create_replicated_table', 'shard1', 'replica2');" -# ) -# assert ( -# "Explicit zookeeper_path and replica_name are specified" -# in main_node.query_and_get_error( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " -# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# ) -# -# assert ( -# "Explicit zookeeper_path and replica_name are specified" -# in main_node.query_and_get_error( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " -# "ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# ) -# -# assert ( -# "This syntax for *MergeTree engine is deprecated" -# in main_node.query_and_get_error( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) " -# "ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);" -# ) -# ) -# -# main_node.query( -# "CREATE TABLE create_replicated_table.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# -# expected = ( -# "CREATE TABLE create_replicated_table.replicated_table\\n(\\n `d` Date,\\n `k` UInt64,\\n `i32` Int32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\n" -# "PARTITION BY toYYYYMM(d)\\nORDER BY k\\nSETTINGS index_granularity = 8192" -# ) -# assert_create_query( -# [main_node, dummy_node], "create_replicated_table.replicated_table", expected -# ) -# # assert without replacing uuid -# assert main_node.query( -# "show create create_replicated_table.replicated_table" -# ) == dummy_node.query("show create create_replicated_table.replicated_table") -# main_node.query("DROP DATABASE create_replicated_table SYNC") -# dummy_node.query("DROP DATABASE create_replicated_table SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_simple_alter_table(started_cluster, engine): -# database = f"test_simple_alter_table_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# # test_simple_alter_table -# name = f"{database}.alter_test" -# main_node.query( -# "CREATE TABLE {} " -# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " -# "ENGINE = {} PARTITION BY StartDate ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);".format( -# name, engine -# ) -# ) -# main_node.query("ALTER TABLE {} ADD COLUMN Added0 UInt32;".format(name)) -# main_node.query("ALTER TABLE {} ADD COLUMN Added2 UInt32;".format(name)) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN Added1 UInt32 AFTER Added0;".format(name) -# ) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;".format( -# name -# ) -# ) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;".format( -# name -# ) -# ) -# main_node.query( -# "ALTER TABLE {} ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;".format( -# name -# ) -# ) -# -# full_engine = ( -# engine -# if not "Replicated" in engine -# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" -# ) -# expected = ( -# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" -# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n" -# " `AddedNested1.A` Array(UInt32),\\n `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n" -# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64)\\n)\\n" -# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" -# "SETTINGS index_granularity = 8192".format(name, full_engine) -# ) -# -# assert_create_query([main_node, dummy_node], name, expected) -# -# # test_create_replica_after_delay -# competing_node.query( -# f"CREATE DATABASE IF NOT EXISTS {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica3');" -# ) -# -# main_node.query("ALTER TABLE {} ADD COLUMN Added3 UInt32;".format(name)) -# main_node.query("ALTER TABLE {} DROP COLUMN AddedNested1;".format(name)) -# main_node.query("ALTER TABLE {} RENAME COLUMN Added1 TO AddedNested1;".format(name)) -# -# full_engine = ( -# engine -# if not "Replicated" in engine -# else engine + "(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')" -# ) -# expected = ( -# "CREATE TABLE {}\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n" -# " `ToDrop` UInt32,\\n `Added0` UInt32,\\n `AddedNested1` UInt32,\\n `Added2` UInt32,\\n" -# " `AddedNested2.A` Array(UInt32),\\n `AddedNested2.B` Array(UInt64),\\n `Added3` UInt32\\n)\\n" -# "ENGINE = {}\\nPARTITION BY StartDate\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\n" -# "SETTINGS index_granularity = 8192".format(name, full_engine) -# ) -# -# assert_create_query([main_node, dummy_node, competing_node], name, expected) -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# competing_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_delete_from_table(started_cluster, engine): -# database = f"delete_from_table_{engine}" -# -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" -# ) -# -# name = f"{database}.delete_test" -# main_node.query( -# "CREATE TABLE {} " -# "(id UInt64, value String) " -# "ENGINE = {} PARTITION BY id%2 ORDER BY (id);".format(name, engine) -# ) -# main_node.query("INSERT INTO TABLE {} VALUES(1, 'aaaa');".format(name)) -# main_node.query("INSERT INTO TABLE {} VALUES(2, 'aaaa');".format(name)) -# dummy_node.query("INSERT INTO TABLE {} VALUES(1, 'bbbb');".format(name)) -# dummy_node.query("INSERT INTO TABLE {} VALUES(2, 'bbbb');".format(name)) -# -# main_node.query("DELETE FROM {} WHERE id=2;".format(name)) -# -# expected = "1\taaaa\n1\tbbbb" -# -# table_for_select = name -# if not "Replicated" in engine: -# table_for_select = f"cluster('{database}', {name})" -# for node in [main_node, dummy_node]: -# assert_eq_with_retry( -# node, -# "SELECT * FROM {} ORDER BY id, value;".format(table_for_select), -# expected, -# ) -# -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# def get_table_uuid(database, name): -# return main_node.query( -# f"SELECT uuid FROM system.tables WHERE database = '{database}' and name = '{name}'" -# ).strip() -# -# -# @pytest.fixture(scope="module", name="attachable_part") -# def fixture_attachable_part(started_cluster): -# main_node.query(f"CREATE DATABASE testdb_attach_atomic ENGINE = Atomic") -# main_node.query( -# f"CREATE TABLE testdb_attach_atomic.test (CounterID UInt32) ENGINE = MergeTree ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO testdb_attach_atomic.test VALUES (123)") -# main_node.query( -# f"ALTER TABLE testdb_attach_atomic.test FREEZE WITH NAME 'test_attach'" -# ) -# table_uuid = get_table_uuid("testdb_attach_atomic", "test") -# return os.path.join( -# main_node.path, -# f"database/shadow/test_attach/store/{table_uuid[:3]}/{table_uuid}/all_1_1_0", -# ) -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_attach(started_cluster, attachable_part, engine): -# database = f"alter_attach_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# main_node.query( -# f"CREATE TABLE {database}.alter_attach_test (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# table_uuid = get_table_uuid(database, "alter_attach_test") -# # Provide and attach a part to the main node -# shutil.copytree( -# attachable_part, -# os.path.join( -# main_node.path, -# f"database/store/{table_uuid[:3]}/{table_uuid}/detached/all_1_1_0", -# ), -# ) -# main_node.query(f"ALTER TABLE {database}.alter_attach_test ATTACH PART 'all_1_1_0'") -# # On the main node, data is attached -# assert ( -# main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") -# == "123\n" -# ) -# # On the other node, data is replicated only if using a Replicated table engine -# if engine == "ReplicatedMergeTree": -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") -# == "123\n" -# ) -# else: -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") -# == "" -# ) -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_drop_part(started_cluster, engine): -# database = f"alter_drop_part_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" -# main_node.query( -# f"CREATE TABLE {database}.alter_drop_part (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (123)") -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_drop_part VALUES (456)") -# else: -# main_node.query(f"SYSTEM SYNC REPLICA {database}.alter_drop_part PULL") -# main_node.query(f"ALTER TABLE {database}.alter_drop_part DROP PART '{part_name}'") -# assert main_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" -# if engine == "ReplicatedMergeTree": -# # The DROP operation is still replicated at the table engine level -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") == "" -# ) -# else: -# assert ( -# dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop_part") -# == "456\n" -# ) -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_detach_part(started_cluster, engine): -# database = f"alter_detach_part_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" -# main_node.query( -# f"CREATE TABLE {database}.alter_detach (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_detach VALUES (123)") -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_detach VALUES (456)") -# main_node.query(f"ALTER TABLE {database}.alter_detach DETACH PART '{part_name}'") -# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_detach'" -# assert main_node.query(detached_parts_query) == f"{part_name}\n" -# if engine == "ReplicatedMergeTree": -# # The detach operation is still replicated at the table engine level -# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" -# else: -# assert dummy_node.query(detached_parts_query) == "" -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_drop_detached_part(started_cluster, engine): -# database = f"alter_drop_detached_part_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# -# part_name = "all_0_0_0" if engine == "ReplicatedMergeTree" else "all_1_1_0" -# main_node.query( -# f"CREATE TABLE {database}.alter_drop_detached (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (123)") -# main_node.query( -# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" -# ) -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_drop_detached VALUES (456)") -# dummy_node.query( -# f"ALTER TABLE {database}.alter_drop_detached DETACH PART '{part_name}'" -# ) -# main_node.query( -# f"ALTER TABLE {database}.alter_drop_detached DROP DETACHED PART '{part_name}'" -# ) -# detached_parts_query = f"SELECT name FROM system.detached_parts WHERE database='{database}' AND table='alter_drop_detached'" -# assert main_node.query(detached_parts_query) == "" -# assert dummy_node.query(detached_parts_query) == f"{part_name}\n" -# -# main_node.query(f"DROP DATABASE {database} SYNC") -# dummy_node.query(f"DROP DATABASE {database} SYNC") -# -# -# @pytest.mark.parametrize("engine", ["MergeTree", "ReplicatedMergeTree"]) -# def test_alter_drop_partition(started_cluster, engine): -# database = f"alter_drop_partition_{engine}" -# main_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard1', 'replica2');" -# ) -# snapshotting_node.query( -# f"CREATE DATABASE {database} ENGINE = Replicated('/test/{database}', 'shard2', 'replica1');" -# ) -# -# main_node.query( -# f"CREATE TABLE {database}.alter_drop (CounterID UInt32) ENGINE = {engine} ORDER BY (CounterID)" -# ) -# main_node.query(f"INSERT INTO {database}.alter_drop VALUES (123)") -# if engine == "MergeTree": -# dummy_node.query(f"INSERT INTO {database}.alter_drop VALUES (456)") -# snapshotting_node.query(f"INSERT INTO {database}.alter_drop VALUES (789)") -# main_node.query( -# f"ALTER TABLE {database}.alter_drop ON CLUSTER {database} DROP PARTITION ID 'all'", -# settings={"replication_alter_partitions_sync": 2}, -# ) -# assert ( -# main_node.query( -# f"SELECT CounterID FROM clusterAllReplicas('{database}', {database}.alter_drop)" -# ) -# == "" -# ) -# assert dummy_node.query(f"SELECT CounterID FROM {database}.alter_drop") == "" -# main_node.query(f"DROP DATABASE {database}") -# dummy_node.query(f"DROP DATABASE {database}") -# snapshotting_node.query(f"DROP DATABASE {database}") -# -# -# def test_alter_fetch(started_cluster): -# main_node.query( -# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE alter_fetch ENGINE = Replicated('/test/alter_fetch', 'shard1', 'replica2');" -# ) -# -# main_node.query( -# "CREATE TABLE alter_fetch.fetch_source (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" -# ) -# main_node.query( -# "CREATE TABLE alter_fetch.fetch_target (CounterID UInt32) ENGINE = ReplicatedMergeTree ORDER BY (CounterID)" -# ) -# main_node.query("INSERT INTO alter_fetch.fetch_source VALUES (123)") -# table_uuid = get_table_uuid("alter_fetch", "fetch_source") -# main_node.query( -# f"ALTER TABLE alter_fetch.fetch_target FETCH PART 'all_0_0_0' FROM '/clickhouse/tables/{table_uuid}/{{shard}}' " -# ) -# detached_parts_query = "SELECT name FROM system.detached_parts WHERE database='alter_fetch' AND table='fetch_target'" -# assert main_node.query(detached_parts_query) == "all_0_0_0\n" -# assert dummy_node.query(detached_parts_query) == "" -# -# main_node.query("DROP DATABASE alter_fetch SYNC") -# dummy_node.query("DROP DATABASE alter_fetch SYNC") -# -# -# def test_alters_from_different_replicas(started_cluster): -# main_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica2');" -# ) -# -# # test_alters_from_different_replicas -# competing_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica3');" -# ) -# -# main_node.query( -# "CREATE TABLE alters_from_different_replicas.concurrent_test " -# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " -# "ENGINE = MergeTree PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID);" -# ) -# -# main_node.query( -# "CREATE TABLE alters_from_different_replicas.dist AS alters_from_different_replicas.concurrent_test ENGINE = Distributed(alters_from_different_replicas, alters_from_different_replicas, concurrent_test, CounterID)" -# ) -# -# dummy_node.stop_clickhouse(kill=True) -# -# settings = {"distributed_ddl_task_timeout": 5} -# assert "is not finished on 1 of 3 hosts" in competing_node.query_and_get_error( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added0 UInt32;", -# settings=settings, -# ) -# settings = { -# "distributed_ddl_task_timeout": 5, -# "distributed_ddl_output_mode": "null_status_on_timeout", -# } -# assert "shard1\treplica2\tQUEUED\t" in main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added2 UInt32;", -# settings=settings, -# ) -# settings = { -# "distributed_ddl_task_timeout": 5, -# "distributed_ddl_output_mode": "never_throw", -# } -# assert "shard1\treplica2\tQUEUED\t" in competing_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN Added1 UInt32 AFTER Added0;", -# settings=settings, -# ) -# dummy_node.start_clickhouse() -# main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1 Nested(A UInt32, B UInt64) AFTER Added2;" -# ) -# competing_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested1.C Array(String) AFTER AddedNested1.B;" -# ) -# main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test ADD COLUMN AddedNested2 Nested(A UInt32, B UInt64) AFTER AddedNested1;" -# ) -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32,\\n" -# " `Added0` UInt32,\\n `Added1` UInt32,\\n `Added2` UInt32,\\n `AddedNested1.A` Array(UInt32),\\n" -# " `AddedNested1.B` Array(UInt64),\\n `AddedNested1.C` Array(String),\\n `AddedNested2.A` Array(UInt32),\\n" -# " `AddedNested2.B` Array(UInt64)\\n)\\n" -# "ENGINE = MergeTree\\nPARTITION BY toYYYYMM(StartDate)\\nORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)\\nSETTINGS index_granularity = 8192" -# ) -# -# assert_create_query( -# [main_node, competing_node], -# "alters_from_different_replicas.concurrent_test", -# expected, -# ) -# -# # test_create_replica_after_delay -# main_node.query("DROP TABLE alters_from_different_replicas.concurrent_test SYNC") -# main_node.query( -# "CREATE TABLE alters_from_different_replicas.concurrent_test " -# "(CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) " -# "ENGINE = ReplicatedMergeTree ORDER BY CounterID;" -# ) -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" -# ) -# -# assert_create_query( -# [main_node, competing_node], -# "alters_from_different_replicas.concurrent_test", -# expected, -# ) -# -# main_node.query( -# "INSERT INTO alters_from_different_replicas.dist (CounterID, StartDate, UserID) SELECT number, addDays(toDate('2020-02-02'), number), intHash32(number) FROM numbers(10)" -# ) -# -# # test_replica_restart -# main_node.restart_clickhouse() -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" -# ) -# -# # test_snapshot_and_snapshot_recover -# snapshotting_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica1');" -# ) -# snapshot_recovering_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard2', 'replica2');" -# ) -# assert_create_query( -# all_nodes, "alters_from_different_replicas.concurrent_test", expected -# ) -# -# main_node.query("SYSTEM FLUSH DISTRIBUTED alters_from_different_replicas.dist") -# main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test UPDATE StartDate = addYears(StartDate, 1) WHERE 1" -# ) -# res = main_node.query( -# "ALTER TABLE alters_from_different_replicas.concurrent_test DELETE WHERE UserID % 2" -# ) -# assert ( -# "shard1\treplica1\tOK" in res -# and "shard1\treplica2\tOK" in res -# and "shard1\treplica3\tOK" in res -# ) -# assert "shard2\treplica1\tOK" in res and "shard2\treplica2\tOK" in res -# -# expected = ( -# "1\t1\tmain_node\n" -# "1\t2\tdummy_node\n" -# "1\t3\tcompeting_node\n" -# "2\t1\tsnapshotting_node\n" -# "2\t2\tsnapshot_recovering_node\n" -# ) -# assert ( -# main_node.query( -# "SELECT shard_num, replica_num, host_name FROM system.clusters WHERE cluster='alters_from_different_replicas'" -# ) -# == expected -# ) -# -# # test_drop_and_create_replica -# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# main_node.query( -# "CREATE DATABASE alters_from_different_replicas ENGINE = Replicated('/test/alters_from_different_replicas', 'shard1', 'replica1');" -# ) -# -# expected = ( -# "CREATE TABLE alters_from_different_replicas.concurrent_test\\n(\\n `CounterID` UInt32,\\n `StartDate` Date,\\n `UserID` UInt32,\\n" -# " `VisitID` UInt32,\\n `NestedColumn.A` Array(UInt8),\\n `NestedColumn.S` Array(String),\\n `ToDrop` UInt32\\n)\\n" -# "ENGINE = ReplicatedMergeTree(\\'/clickhouse/tables/{uuid}/{shard}\\', \\'{replica}\\')\\nORDER BY CounterID\\nSETTINGS index_granularity = 8192" -# ) -# -# assert_create_query( -# [main_node, competing_node], -# "alters_from_different_replicas.concurrent_test", -# expected, -# ) -# assert_create_query( -# all_nodes, "alters_from_different_replicas.concurrent_test", expected -# ) -# -# for node in all_nodes: -# node.query("SYSTEM SYNC REPLICA alters_from_different_replicas.concurrent_test") -# -# expected = ( -# "0\t2021-02-02\t4249604106\n" -# "1\t2021-02-03\t1343103100\n" -# "4\t2021-02-06\t3902320246\n" -# "7\t2021-02-09\t3844986530\n" -# "9\t2021-02-11\t1241149650\n" -# ) -# -# assert_eq_with_retry( -# dummy_node, -# "SELECT CounterID, StartDate, UserID FROM alters_from_different_replicas.dist ORDER BY CounterID", -# expected, -# ) -# main_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# dummy_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# competing_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# snapshotting_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# snapshot_recovering_node.query("DROP DATABASE alters_from_different_replicas SYNC") -# -# -# def create_some_tables(db): -# settings = { -# "distributed_ddl_task_timeout": 0, -# "allow_experimental_object_type": 1, -# "allow_suspicious_codecs": 1, -# } -# main_node.query(f"CREATE TABLE {db}.t1 (n int) ENGINE=Memory", settings=settings) -# dummy_node.query( -# f"CREATE TABLE {db}.t2 (s String) ENGINE=Memory", settings=settings -# ) -# main_node.query( -# f"CREATE TABLE {db}.mt1 (n int) ENGINE=MergeTree order by n", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE TABLE {db}.mt2 (n int) ENGINE=MergeTree order by n", -# settings=settings, -# ) -# main_node.query( -# f"CREATE TABLE {db}.rmt1 (n int) ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE TABLE {db}.rmt2 (n int CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12))) ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# main_node.query( -# f"CREATE TABLE {db}.rmt3 (n int, json Object('json') materialized '') ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE TABLE {db}.rmt5 (n int) ENGINE=ReplicatedMergeTree order by n", -# settings=settings, -# ) -# main_node.query( -# f"CREATE MATERIALIZED VIEW {db}.mv1 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt1", -# settings=settings, -# ) -# dummy_node.query( -# f"CREATE MATERIALIZED VIEW {db}.mv2 (n int) ENGINE=ReplicatedMergeTree order by n AS SELECT n FROM recover.rmt2", -# settings=settings, -# ) -# main_node.query( -# f"CREATE DICTIONARY {db}.d1 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " -# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " -# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" -# ) -# dummy_node.query( -# f"CREATE DICTIONARY {db}.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " -# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt2' PASSWORD '' DB 'recover')) " -# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())" -# ) -# -# -# # These tables are used to check that DatabaseReplicated correctly renames all the tables in case when it restores from the lost state -# def create_table_for_exchanges(db): -# settings = {"distributed_ddl_task_timeout": 0} -# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: -# main_node.query( -# f"CREATE TABLE {db}.{table} (s String) ENGINE=ReplicatedMergeTree order by s", -# settings=settings, -# ) -# -# -# def test_recover_staled_replica(started_cluster): -# main_node.query( -# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica1');" -# ) -# started_cluster.get_kazoo_client("zoo1").set( -# "/clickhouse/databases/recover/logs_to_keep", b"10" -# ) -# dummy_node.query( -# "CREATE DATABASE recover ENGINE = Replicated('/clickhouse/databases/recover', 'shard1', 'replica2');" -# ) -# -# settings = {"distributed_ddl_task_timeout": 0} -# create_some_tables("recover") -# create_table_for_exchanges("recover") -# -# for table in ["t1", "t2", "mt1", "mt2", "rmt1", "rmt2", "rmt3", "rmt5"]: -# main_node.query(f"INSERT INTO recover.{table} VALUES (42)") -# for table in ["t1", "t2", "mt1", "mt2"]: -# dummy_node.query(f"INSERT INTO recover.{table} VALUES (42)") -# -# for i, table in enumerate(["a1", "a2", "a3", "a4", "a5", "a6"]): -# main_node.query(f"INSERT INTO recover.{table} VALUES ('{str(i + 1) * 10}')") -# -# for table in ["rmt1", "rmt2", "rmt3", "rmt5"]: -# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") -# for table in ["a1", "a2", "a3", "a4", "a5", "a6"]: -# main_node.query(f"SYSTEM SYNC REPLICA recover.{table}") -# -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(dummy_node) -# dummy_node.query_and_get_error("RENAME TABLE recover.t1 TO recover.m1") -# -# main_node.query_with_retry( -# "RENAME TABLE recover.t1 TO recover.m1", settings=settings -# ) -# main_node.query_with_retry( -# "ALTER TABLE recover.mt1 ADD COLUMN m int", settings=settings -# ) -# main_node.query_with_retry( -# "ALTER TABLE recover.rmt1 ADD COLUMN m int", settings=settings -# ) -# main_node.query_with_retry( -# "RENAME TABLE recover.rmt3 TO recover.rmt4", settings=settings -# ) -# main_node.query_with_retry("DROP TABLE recover.rmt5", settings=settings) -# main_node.query_with_retry("DROP DICTIONARY recover.d2", settings=settings) -# main_node.query_with_retry( -# "CREATE DICTIONARY recover.d2 (n int DEFAULT 0, m int DEFAULT 1) PRIMARY KEY n " -# "SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'rmt1' PASSWORD '' DB 'recover')) " -# "LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());", -# settings=settings, -# ) -# -# inner_table = ( -# ".inner_id." -# + dummy_node.query_with_retry( -# "SELECT uuid FROM system.tables WHERE database='recover' AND name='mv1'" -# ).strip() -# ) -# main_node.query_with_retry( -# f"ALTER TABLE recover.`{inner_table}` MODIFY COLUMN n int DEFAULT 42", -# settings=settings, -# ) -# main_node.query_with_retry( -# "ALTER TABLE recover.mv1 MODIFY QUERY SELECT m as n FROM recover.rmt1", -# settings=settings, -# ) -# main_node.query_with_retry( -# "RENAME TABLE recover.mv2 TO recover.mv3", -# settings=settings, -# ) -# -# main_node.query_with_retry( -# "CREATE TABLE recover.tmp AS recover.m1", settings=settings -# ) -# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) -# main_node.query_with_retry( -# "CREATE TABLE recover.tmp AS recover.m1", settings=settings -# ) -# main_node.query_with_retry("DROP TABLE recover.tmp", settings=settings) -# main_node.query_with_retry( -# "CREATE TABLE recover.tmp AS recover.m1", settings=settings -# ) -# -# main_node.query("EXCHANGE TABLES recover.a1 AND recover.a2", settings=settings) -# main_node.query("EXCHANGE TABLES recover.a3 AND recover.a4", settings=settings) -# main_node.query("EXCHANGE TABLES recover.a5 AND recover.a4", settings=settings) -# main_node.query("EXCHANGE TABLES recover.a6 AND recover.a3", settings=settings) -# main_node.query("RENAME TABLE recover.a6 TO recover.a7", settings=settings) -# main_node.query("RENAME TABLE recover.a1 TO recover.a8", settings=settings) -# -# assert ( -# main_node.query( -# "SELECT name FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' ORDER BY name" -# ) -# == "a2\na3\na4\na5\na7\na8\nd1\nd2\nm1\nmt1\nmt2\nmv1\nmv3\nrmt1\nrmt2\nrmt4\nt2\ntmp\n" -# ) -# query = ( -# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover' AND name NOT LIKE '.inner_id.%' " -# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" -# ) -# expected = main_node.query(query) -# assert_eq_with_retry(dummy_node, query, expected) -# assert ( -# main_node.query( -# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" -# ) -# == "2\n" -# ) -# assert ( -# dummy_node.query( -# "SELECT count() FROM system.tables WHERE database='recover' AND name LIKE '.inner_id.%'" -# ) -# == "2\n" -# ) -# -# # Check that Database Replicated renamed all the tables correctly -# for i, table in enumerate(["a2", "a8", "a5", "a7", "a4", "a3"]): -# assert ( -# dummy_node.query(f"SELECT * FROM recover.{table}") == f"{str(i + 1) * 10}\n" -# ) -# -# for table in [ -# "m1", -# "t2", -# "mt1", -# "mt2", -# "rmt1", -# "rmt2", -# "rmt4", -# "d1", -# "d2", -# "mv1", -# "mv3", -# ]: -# assert main_node.query(f"SELECT (*,).1 FROM recover.{table}") == "42\n" -# for table in ["t2", "rmt1", "rmt2", "rmt4", "d1", "d2", "mt2", "mv1", "mv3"]: -# assert ( -# dummy_node.query(f"SELECT '{table}', (*,).1 FROM recover.{table}") -# == f"{table}\t42\n" -# ) -# for table in ["m1", "mt1"]: -# assert dummy_node.query(f"SELECT count() FROM recover.{table}") == "0\n" -# global test_recover_staled_replica_run -# assert ( -# dummy_node.query( -# "SELECT count() FROM system.tables WHERE database='recover_broken_tables'" -# ) -# == f"{test_recover_staled_replica_run}\n" -# ) -# assert ( -# dummy_node.query( -# "SELECT count() FROM system.tables WHERE database='recover_broken_replicated_tables'" -# ) -# == f"{test_recover_staled_replica_run}\n" -# ) -# test_recover_staled_replica_run += 1 -# -# print(dummy_node.query("SHOW DATABASES")) -# print(dummy_node.query("SHOW TABLES FROM recover_broken_tables")) -# print(dummy_node.query("SHOW TABLES FROM recover_broken_replicated_tables")) -# -# table = dummy_node.query( -# "SHOW TABLES FROM recover_broken_tables LIKE 'mt1_41_%' LIMIT 1" -# ).strip() -# assert ( -# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_tables.{table}") == "42\n" -# ) -# table = dummy_node.query( -# "SHOW TABLES FROM recover_broken_replicated_tables LIKE 'rmt5_41_%' LIMIT 1" -# ).strip() -# assert ( -# dummy_node.query(f"SELECT (*,).1 FROM recover_broken_replicated_tables.{table}") -# == "42\n" -# ) -# -# expected = "Cleaned 6 outdated objects: dropped 1 dictionaries and 3 tables, moved 2 tables" -# assert_logs_contain(dummy_node, expected) -# -# dummy_node.query("DROP TABLE recover.tmp") -# assert_eq_with_retry( -# main_node, -# "SELECT count() FROM system.tables WHERE database='recover' AND name='tmp'", -# "0\n", -# ) -# main_node.query("DROP DATABASE recover SYNC") -# dummy_node.query("DROP DATABASE recover SYNC") -# -# -# def test_recover_staled_replica_many_mvs(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS recover_mvs") -# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") -# -# main_node.query_with_retry( -# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica1');" -# ) -# started_cluster.get_kazoo_client("zoo1").set( -# "/clickhouse/databases/recover_mvs/logs_to_keep", b"10" -# ) -# dummy_node.query_with_retry( -# "CREATE DATABASE IF NOT EXISTS recover_mvs ENGINE = Replicated('/clickhouse/databases/recover_mvs', 'shard1', 'replica2');" -# ) -# -# settings = {"distributed_ddl_task_timeout": 0} -# -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(dummy_node) -# dummy_node.query_and_get_error("RENAME TABLE recover_mvs.t1 TO recover_mvs.m1") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query( -# f"CREATE TABLE recover_mvs.rmt{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", -# settings=settings, -# ) -# -# print("Created tables") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query( -# f"CREATE TABLE recover_mvs.mv_inner{identifier} (n int) ENGINE=ReplicatedMergeTree ORDER BY n", -# settings=settings, -# ) -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE MATERIALIZED VIEW recover_mvs.mv{identifier} -# TO recover_mvs.mv_inner{identifier} -# AS SELECT * FROM recover_mvs.rmt{identifier}""", -# settings=settings, -# ) -# -# print("Created MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.view_from_mv{identifier} -# AS SELECT * FROM recover_mvs.mv{identifier}""", -# settings=settings, -# ) -# -# print("Created Views on top of MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE MATERIALIZED VIEW recover_mvs.cascade_mv{identifier} -# ENGINE=MergeTree() ORDER BY tuple() -# POPULATE AS SELECT * FROM recover_mvs.mv_inner{identifier};""", -# settings=settings, -# ) -# -# print("Created cascade MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.view_from_cascade_mv{identifier} -# AS SELECT * FROM recover_mvs.cascade_mv{identifier}""", -# settings=settings, -# ) -# -# print("Created Views on top of cascade MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE MATERIALIZED VIEW recover_mvs.double_cascade_mv{identifier} -# ENGINE=MergeTree() ORDER BY tuple() -# POPULATE AS SELECT * FROM recover_mvs.`.inner_id.{get_table_uuid("recover_mvs", f"cascade_mv{identifier}")}`""", -# settings=settings, -# ) -# -# print("Created double cascade MVs") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.view_from_double_cascade_mv{identifier} -# AS SELECT * FROM recover_mvs.double_cascade_mv{identifier}""", -# settings=settings, -# ) -# -# print("Created Views on top of double cascade MVs") -# -# # This weird table name is actually makes sence because it starts with letter `a` and may break some internal sorting -# main_node.query_with_retry( -# """ -# CREATE VIEW recover_mvs.anime -# AS -# SELECT n -# FROM -# ( -# SELECT * -# FROM -# ( -# SELECT * -# FROM -# ( -# SELECT * -# FROM recover_mvs.mv_inner1 AS q1 -# INNER JOIN recover_mvs.mv_inner2 AS q2 ON q1.n = q2.n -# ) AS new_table_1 -# INNER JOIN recover_mvs.mv_inner3 AS q3 ON new_table_1.n = q3.n -# ) AS new_table_2 -# INNER JOIN recover_mvs.mv_inner4 AS q4 ON new_table_2.n = q4.n -# ) -# """, -# settings=settings, -# ) -# -# print("Created final boss") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE DICTIONARY recover_mvs.`11111d{identifier}` (n UInt64) -# PRIMARY KEY n -# SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE 'double_cascade_mv{identifier}' DB 'recover_mvs')) -# LAYOUT(FLAT()) LIFETIME(1)""", -# settings=settings, -# ) -# -# print("Created dictionaries") -# -# for identifier in ["1", "2", "3", "4"]: -# main_node.query_with_retry( -# f"""CREATE VIEW recover_mvs.`00000vd{identifier}` -# AS SELECT * FROM recover_mvs.`11111d{identifier}`""", -# settings=settings, -# ) -# -# print("Created Views on top of dictionaries") -# -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_mvs") -# query = "SELECT name FROM system.tables WHERE database='recover_mvs' ORDER BY name" -# assert main_node.query(query) == dummy_node.query(query) -# -# main_node.query("DROP DATABASE IF EXISTS recover_mvs") -# dummy_node.query("DROP DATABASE IF EXISTS recover_mvs") -# -# -# def test_startup_without_zk(started_cluster): -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(main_node) -# err = main_node.query_and_get_error( -# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" -# ) -# assert "ZooKeeper" in err or "Coordination::Exception" in err -# main_node.query( -# "CREATE DATABASE startup ENGINE = Replicated('/clickhouse/databases/startup', 'shard1', 'replica1');" -# ) -# main_node.query( -# "CREATE TABLE startup.rmt (n int) ENGINE=ReplicatedMergeTree order by n" -# ) -# -# main_node.query("INSERT INTO startup.rmt VALUES (42)") -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(main_node) -# main_node.restart_clickhouse(stop_start_wait_sec=60) -# assert main_node.query("SELECT (*,).1 FROM startup.rmt") == "42\n" -# -# # we need to wait until the table is not readonly -# main_node.query_with_retry("INSERT INTO startup.rmt VALUES(42)") -# -# main_node.query_with_retry("CREATE TABLE startup.m (n int) ENGINE=Memory") -# -# main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") -# assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" -# -# main_node.query("DROP DATABASE startup SYNC") -# -# -# def test_server_uuid(started_cluster): -# uuid1 = main_node.query("select serverUUID()") -# uuid2 = dummy_node.query("select serverUUID()") -# assert uuid1 != uuid2 -# main_node.restart_clickhouse() -# uuid1_after_restart = main_node.query("select serverUUID()") -# assert uuid1 == uuid1_after_restart -# -# -# def test_sync_replica(started_cluster): -# main_node.query( -# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE test_sync_database ENGINE = Replicated('/test/sync_replica', 'shard1', 'replica2');" -# ) -# -# number_of_tables = 1000 -# -# settings = {"distributed_ddl_task_timeout": 0} -# -# with PartitionManager() as pm: -# pm.drop_instance_zk_connections(dummy_node) -# -# for i in range(number_of_tables): -# main_node.query( -# "CREATE TABLE test_sync_database.table_{} (n int) ENGINE=MergeTree order by n".format( -# i -# ), -# settings=settings, -# ) -# -# # wait for host to reconnect -# dummy_node.query_with_retry("SELECT * FROM system.zookeeper WHERE path='/'") -# -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_sync_database") -# -# assert "2\n" == main_node.query( -# "SELECT sum(is_active) FROM system.clusters WHERE cluster='test_sync_database'" -# ) -# -# assert dummy_node.query( -# "SELECT count() FROM system.tables where database='test_sync_database'" -# ).strip() == str(number_of_tables) -# -# assert main_node.query( -# "SELECT count() FROM system.tables where database='test_sync_database'" -# ).strip() == str(number_of_tables) -# -# engine_settings = {"default_table_engine": "ReplicatedMergeTree"} -# dummy_node.query( -# "CREATE TABLE test_sync_database.table (n int, primary key n) partition by n", -# settings=engine_settings, -# ) -# main_node.query("INSERT INTO test_sync_database.table SELECT * FROM numbers(10)") -# dummy_node.query("TRUNCATE TABLE test_sync_database.table", settings=settings) -# dummy_node.query( -# "ALTER TABLE test_sync_database.table ADD COLUMN m int", settings=settings -# ) -# -# main_node.query( -# "SYSTEM SYNC DATABASE REPLICA ON CLUSTER test_sync_database test_sync_database" -# ) -# -# lp1 = main_node.query( -# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica1' and name='log_ptr'" -# ) -# lp2 = main_node.query( -# "select value from system.zookeeper where path='/test/sync_replica/replicas/shard1|replica2' and name='log_ptr'" -# ) -# max_lp = main_node.query( -# "select value from system.zookeeper where path='/test/sync_replica/' and name='max_log_ptr'" -# ) -# assert lp1 == max_lp -# assert lp2 == max_lp -# -# main_node.query("DROP DATABASE test_sync_database SYNC") -# dummy_node.query("DROP DATABASE test_sync_database SYNC") -# -# -# def test_force_synchronous_settings(started_cluster): -# main_node.query( -# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard1', 'replica2');" -# ) -# snapshotting_node.query( -# "CREATE DATABASE test_force_synchronous_settings ENGINE = Replicated('/clickhouse/databases/test2', 'shard2', 'replica1');" -# ) -# main_node.query( -# "CREATE TABLE test_force_synchronous_settings.t (n int) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" -# ) -# main_node.query( -# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" -# ) -# snapshotting_node.query( -# "INSERT INTO test_force_synchronous_settings.t SELECT * FROM numbers(10)" -# ) -# snapshotting_node.query( -# "SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings" -# ) -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA test_force_synchronous_settings") -# -# snapshotting_node.query("SYSTEM STOP MERGES test_force_synchronous_settings.t") -# -# def start_merges_func(): -# time.sleep(5) -# snapshotting_node.query("SYSTEM START MERGES test_force_synchronous_settings.t") -# -# start_merges_thread = threading.Thread(target=start_merges_func) -# start_merges_thread.start() -# -# settings = { -# "mutations_sync": 2, -# "database_replicated_enforce_synchronous_settings": 1, -# } -# main_node.query( -# "ALTER TABLE test_force_synchronous_settings.t UPDATE n = n * 10 WHERE 1", -# settings=settings, -# ) -# assert "10\t450\n" == snapshotting_node.query( -# "SELECT count(), sum(n) FROM test_force_synchronous_settings.t" -# ) -# start_merges_thread.join() -# -# def select_func(): -# dummy_node.query( -# "SELECT sleepEachRow(1) FROM test_force_synchronous_settings.t SETTINGS function_sleep_max_microseconds_per_block = 0" -# ) -# -# select_thread = threading.Thread(target=select_func) -# select_thread.start() -# -# settings = {"database_replicated_enforce_synchronous_settings": 1} -# snapshotting_node.query( -# "DROP TABLE test_force_synchronous_settings.t SYNC", settings=settings -# ) -# main_node.query( -# "CREATE TABLE test_force_synchronous_settings.t (n String) ENGINE=ReplicatedMergeTree('/test/same/path/{shard}', '{replica}') ORDER BY tuple()" -# ) -# select_thread.join() -# -# -# def test_recover_digest_mismatch(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# -# main_node.query( -# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE recover_digest_mismatch ENGINE = Replicated('/clickhouse/databases/recover_digest_mismatch', 'shard1', 'replica2');" -# ) -# -# create_some_tables("recover_digest_mismatch") -# -# main_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA recover_digest_mismatch") -# -# ways_to_corrupt_metadata = [ -# "mv /var/lib/clickhouse/metadata/recover_digest_mismatch/t1.sql /var/lib/clickhouse/metadata/recover_digest_mismatch/m1.sql", -# "sed --follow-symlinks -i 's/Int32/String/' /var/lib/clickhouse/metadata/recover_digest_mismatch/mv1.sql", -# "rm -f /var/lib/clickhouse/metadata/recover_digest_mismatch/d1.sql", -# "rm -rf /var/lib/clickhouse/metadata/recover_digest_mismatch/", # Will trigger "Directory already exists" -# "rm -rf /var/lib/clickhouse/store", -# ] -# -# for command in ways_to_corrupt_metadata: -# print(f"Corrupting data using `{command}`") -# need_remove_is_active_node = "rm -rf" in command -# dummy_node.stop_clickhouse(kill=not need_remove_is_active_node) -# dummy_node.exec_in_container(["bash", "-c", command]) -# -# query = ( -# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='recover_digest_mismatch' AND name NOT LIKE '.inner_id.%' " -# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" -# ) -# expected = main_node.query(query) -# -# if need_remove_is_active_node: -# # NOTE Otherwise it fails to recreate ReplicatedMergeTree table due to "Replica already exists" -# main_node.query( -# "SYSTEM DROP REPLICA '2' FROM DATABASE recover_digest_mismatch" -# ) -# -# # There is a race condition between deleting active node and creating it on server startup -# # So we start a server only after we deleted all table replicas from the Keeper -# dummy_node.start_clickhouse() -# assert_eq_with_retry(dummy_node, query, expected) -# -# main_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# dummy_node.query("DROP DATABASE IF EXISTS recover_digest_mismatch") -# -# print("Everything Okay") -# -# -# def test_replicated_table_structure_alter(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS table_structure") -# dummy_node.query("DROP DATABASE IF EXISTS table_structure") -# -# main_node.query( -# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica2');" -# ) -# competing_node.query( -# "CREATE DATABASE table_structure ENGINE = Replicated('/clickhouse/databases/table_structure', 'shard1', 'replica3');" -# ) -# -# competing_node.query("CREATE TABLE table_structure.mem (n int) ENGINE=Memory") -# dummy_node.query("DETACH DATABASE table_structure") -# -# settings = {"distributed_ddl_task_timeout": 0} -# main_node.query( -# "CREATE TABLE table_structure.rmt (n int, v UInt64) ENGINE=ReplicatedReplacingMergeTree(v) ORDER BY n", -# settings=settings, -# ) -# -# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# competing_node.query("DETACH DATABASE table_structure") -# -# main_node.query( -# "ALTER TABLE table_structure.rmt ADD COLUMN m int", settings=settings -# ) -# main_node.query( -# "ALTER TABLE table_structure.rmt COMMENT COLUMN v 'version'", settings=settings -# ) -# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3)") -# -# command = "rm -f /var/lib/clickhouse/metadata/table_structure/mem.sql" -# competing_node.exec_in_container(["bash", "-c", command]) -# competing_node.restart_clickhouse(kill=True) -# -# dummy_node.query("ATTACH DATABASE table_structure") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") -# assert "1\t2\t3\n" == dummy_node.query("SELECT * FROM table_structure.rmt") -# -# competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# competing_node.query("SYSTEM SYNC REPLICA table_structure.rmt") -# # time.sleep(600) -# assert "mem" in competing_node.query("SHOW TABLES FROM table_structure") -# assert "1\t2\t3\n" == competing_node.query("SELECT * FROM table_structure.rmt") -# -# main_node.query("ALTER TABLE table_structure.rmt ADD COLUMN k int") -# main_node.query("INSERT INTO table_structure.rmt VALUES (1, 2, 3, 4)") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") -# dummy_node.query("SYSTEM SYNC REPLICA table_structure.rmt") -# assert "1\t2\t3\t0\n1\t2\t3\t4\n" == dummy_node.query( -# "SELECT * FROM table_structure.rmt ORDER BY k" -# ) -# -# -# def test_modify_comment(started_cluster): -# main_node.query( -# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '1');" -# ) -# -# dummy_node.query( -# "CREATE DATABASE modify_comment_db ENGINE = Replicated('/test/modify_comment', 'shard1', 'replica' || '2');" -# ) -# -# main_node.query( -# "CREATE TABLE modify_comment_db.modify_comment_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);" -# ) -# -# def restart_verify_not_readonly(): -# main_node.restart_clickhouse() -# assert ( -# main_node.query( -# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" -# ) -# == "0\n" -# ) -# dummy_node.restart_clickhouse() -# assert ( -# dummy_node.query( -# "SELECT is_readonly FROM system.replicas WHERE table = 'modify_comment_table'" -# ) -# == "0\n" -# ) -# -# main_node.query( -# "ALTER TABLE modify_comment_db.modify_comment_table COMMENT COLUMN d 'Some comment'" -# ) -# -# restart_verify_not_readonly() -# -# main_node.query( -# "ALTER TABLE modify_comment_db.modify_comment_table MODIFY COMMENT 'Some error comment'" -# ) -# -# restart_verify_not_readonly() -# -# main_node.query("DROP DATABASE modify_comment_db SYNC") -# dummy_node.query("DROP DATABASE modify_comment_db SYNC") -# -# -# def test_table_metadata_corruption(started_cluster): -# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") -# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") -# -# main_node.query( -# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica1');" -# ) -# dummy_node.query( -# "CREATE DATABASE table_metadata_corruption ENGINE = Replicated('/clickhouse/databases/table_metadata_corruption', 'shard1', 'replica2');" -# ) -# -# create_some_tables("table_metadata_corruption") -# -# main_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") -# dummy_node.query("SYSTEM SYNC DATABASE REPLICA table_metadata_corruption") -# -# # Server should handle this by throwing an exception during table loading, which should lead to server shutdown -# corrupt = "sed --follow-symlinks -i 's/ReplicatedMergeTree/CorruptedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" -# -# print(f"Corrupting metadata using `{corrupt}`") -# dummy_node.stop_clickhouse(kill=True) -# dummy_node.exec_in_container(["bash", "-c", corrupt]) -# -# query = ( -# "SELECT name, uuid, create_table_query FROM system.tables WHERE database='table_metadata_corruption' AND name NOT LIKE '.inner_id.%' " -# "ORDER BY name SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1" -# ) -# expected = main_node.query(query) -# -# # We expect clickhouse server to shutdown without LOGICAL_ERRORs or deadlocks -# dummy_node.start_clickhouse(expected_to_fail=True) -# assert not dummy_node.contains_in_log("LOGICAL_ERROR") -# -# fix_corrupt = "sed --follow-symlinks -i 's/CorruptedMergeTree/ReplicatedMergeTree/' /var/lib/clickhouse/metadata/table_metadata_corruption/rmt1.sql" -# print(f"Fix corrupted metadata using `{fix_corrupt}`") -# dummy_node.exec_in_container(["bash", "-c", fix_corrupt]) -# -# dummy_node.start_clickhouse() -# assert_eq_with_retry(dummy_node, query, expected) -# -# main_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") -# dummy_node.query("DROP DATABASE IF EXISTS table_metadata_corruption") diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py index ac15333cd11..7378598e85e 100644 --- a/tests/integration/test_refreshable_mat_view/test_schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/test_schedule_model.py @@ -4,87 +4,78 @@ from test_refreshable_mat_view.schedule_model import get_next_refresh_time def test_refresh_schedules(): - time_ = datetime(2000, 1, 1, 1, 1, 1) + t = datetime(2000, 1, 1, 1, 1, 1) - assert get_next_refresh_time( - "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", time_ - ) == datetime(2000, 1, 1, 1, 1, 1) - - assert get_next_refresh_time("EVERY 1 SECOND", time_) == datetime( - 2000, 1, 1, 1, 1, 2 - ) - assert get_next_refresh_time("EVERY 1 MINUTE", time_) == datetime( - 2000, - 1, - 1, - 1, - 2, - ) - assert get_next_refresh_time("EVERY 1 HOUR", time_) == datetime( - 2000, - 1, - 1, - 2, - ) - assert get_next_refresh_time("EVERY 1 DAY", time_) == datetime(2000, 1, 2) - assert get_next_refresh_time("EVERY 1 WEEK", time_) == datetime(2000, 1, 10) - assert get_next_refresh_time("EVERY 2 WEEK", time_) == datetime(2000, 1, 17) - assert get_next_refresh_time("EVERY 1 MONTH", time_) == datetime(2000, 2, 1) - assert get_next_refresh_time("EVERY 1 YEAR", time_) == datetime(2001, 1, 1) - - assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", time_) == datetime( - 2003, 5, 11 - ) - - # OFFSET - assert get_next_refresh_time( - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ - ) == datetime(2000, 2, 6, 2, 30, 15) - assert get_next_refresh_time( - "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", time_ - ) == datetime(2001, 3, 6, 2, 30, 15) - - assert get_next_refresh_time( - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", time_ - ) == datetime(2000, 1, 22, 15, 10) - - # AFTER - assert get_next_refresh_time("AFTER 30 SECOND", time_) == datetime( - 2000, 1, 1, 1, 1, 31 - ) - assert get_next_refresh_time("AFTER 30 MINUTE", time_) == datetime( - 2000, 1, 1, 1, 31, 1 - ) - assert get_next_refresh_time("AFTER 2 HOUR", time_) == datetime(2000, 1, 1, 3, 1, 1) - assert get_next_refresh_time("AFTER 2 DAY", time_) == datetime(2000, 1, 3, 1, 1, 1) - assert get_next_refresh_time("AFTER 2 WEEK", time_) == datetime( - 2000, 1, 15, 1, 1, 1 - ) - assert get_next_refresh_time("AFTER 2 MONTH", time_) == datetime( - 2000, 3, 1, 1, 1, 1 - ) - assert get_next_refresh_time("AFTER 2 YEAR", time_) == datetime(2002, 1, 1, 1, 1, 1) - - assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", time_) == datetime( - 2002, 2, 1, 1, 1, 1 - ) - - assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", time_) == datetime( - 2002, 2, 1, 1, 1, 1 - ) + # assert get_next_refresh_time("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) + # assert get_next_refresh_time("EVERY 1 MINUTE", t) == datetime( + # 2000, + # 1, + # 1, + # 1, + # 2, + # ) + # assert get_next_refresh_time("EVERY 1 HOUR", t) == datetime( + # 2000, + # 1, + # 1, + # 2, + # ) + # assert get_next_refresh_time("EVERY 1 DAY", t) == datetime(2000, 1, 2) + # assert get_next_refresh_time("EVERY 1 WEEK", t) == datetime(2000, 1, 10) + # assert get_next_refresh_time("EVERY 2 WEEK", t) == datetime(2000, 1, 17) + # assert get_next_refresh_time("EVERY 1 MONTH", t) == datetime(2000, 2, 1) + # assert get_next_refresh_time("EVERY 1 YEAR", t) == datetime(2001, 1, 1) + # + # assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( + # 2003, 5, 11 + # ) + # + # # OFFSET + # assert get_next_refresh_time( + # "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + # ) == datetime(2000, 2, 6, 2, 30, 15) + # assert get_next_refresh_time( + # "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + # ) == datetime(2001, 3, 6, 2, 30, 15) + # + # assert get_next_refresh_time( + # "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t + # ) == datetime(2000, 1, 22, 15, 10) + # + # # AFTER + # assert get_next_refresh_time("AFTER 30 SECOND", t) == datetime(2000, 1, 1, 1, 1, 31) + # assert get_next_refresh_time("AFTER 30 MINUTE", t) == datetime(2000, 1, 1, 1, 31, 1) + # assert get_next_refresh_time("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) + # assert get_next_refresh_time("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) + # assert get_next_refresh_time("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) + # assert get_next_refresh_time("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) + # assert get_next_refresh_time("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) + # + # assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", t) == datetime( + # 2002, 2, 1, 1, 1, 1 + # ) + # + # assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", t) == datetime( + # 2002, 2, 1, 1, 1, 1 + # ) # RANDOMIZE next_refresh = get_next_refresh_time( - "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", time_ + "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", t ) assert next_refresh == (datetime(2000, 1, 2, 2, 0), datetime(2000, 1, 2, 3, 0)) next_refresh = get_next_refresh_time( "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", - time_, + t, ) assert next_refresh == ( - datetime(2000, 3, 4, 11, 0, 20), - datetime(2000, 3, 7, 12, 0, 20), + datetime(2000, 3, 4, 8, 0, 20), + datetime(2000, 3, 7, 9, 0, 20), + ) + + assert get_next_refresh_time("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( + datetime(2000, 3, 4, 1, 1, 1), + datetime(2000, 3, 5, 1, 1, 1), ) From 6d72f4774e35b1f80c410a75ee10a76af45c41d1 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Sat, 10 Aug 2024 08:11:35 +0200 Subject: [PATCH 26/38] Tests: add RMV tests --- .../sql-reference/statements/create/view.md | 2 +- tests/integration/helpers/test_tools.py | 13 + .../schedule_model.py | 90 +- .../test_refreshable_mat_view/test.py | 1553 +++++++---------- .../test_schedule_model.py | 128 +- 5 files changed, 745 insertions(+), 1041 deletions(-) diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index a6f9f394871..a8be515765b 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -282,7 +282,7 @@ This replaces *all* refresh parameters at once: schedule, dependencies, settings The status of all refreshable materialized views is available in table [`system.view_refreshes`](../../../operations/system-tables/view_refreshes.md). In particular, it contains refresh progress (if running), last and next refresh time, exception message if a refresh failed. -To manually stop, start, trigger, or cancel refreshes use [`SYSTEM STOP|START|REFRESH|CANCEL VIEW`](../system.md#refreshable-materialized-views). +To manually stop, start, trigger, or cancel refreshes use [`SYSTEM STOP|START|REFRESH|WAIT|CANCEL VIEW`](../system.md#refreshable-materialized-views). To wait for a refresh to complete, use [`SYSTEM WAIT VIEW`](../system.md#refreshable-materialized-views). In particular, useful for waiting for initial refresh after creating a view. diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index e71698692be..c334a7366da 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -182,3 +182,16 @@ def csv_compare(result, expected): mismatch.append("+[%d]=%s" % (i, csv_result.lines[i])) return "\n".join(mismatch) + + +def wait_condition(func, condition, max_attempts=10, delay=0.1): + attempts = 0 + while attempts < max_attempts: + result = func() + if condition(result): + return result + attempts += 1 + if attempts < max_attempts: + time.sleep(delay) + + raise Exception(f"Function did not satisfy condition after {max_attempts} attempts") diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py index 5195df3dec6..38781e59522 100644 --- a/tests/integration/test_refreshable_mat_view/schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/schedule_model.py @@ -16,11 +16,8 @@ def relative_offset(unit, value): return rd.relativedelta(hours=value) elif unit == "DAY": return rd.relativedelta(days=value) - # elif unit == "WEEK": - # return rd.relativedelta(days=7 * value) elif unit == "WEEK": return rd.relativedelta(weeks=7 * value) - elif unit == "MONTH": return rd.relativedelta(months=value) elif unit == "YEAR": @@ -42,7 +39,7 @@ def group_and_sort(parts, reverse=False): return sorted_parts -def get_next_refresh_time(schedule, current_time: datetime): +def get_next_refresh_time(schedule, current_time: datetime, first_week=False): parts = schedule.split() randomize_offset = timedelta() @@ -65,6 +62,7 @@ def get_next_refresh_time(schedule, current_time: datetime): parts = parts[:offset_index] + week_in_primary = False if parts[0] == "EVERY": parts = group_and_sort(parts[1:]) for part in parts: @@ -88,9 +86,10 @@ def get_next_refresh_time(schedule, current_time: datetime): hour=0, minute=0, second=0, microsecond=0 ) + rd.relativedelta(days=value) elif unit == "WEEK": + week_in_primary = True current_time = current_time.replace( hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(weekday=0, weeks=value) + ) + rd.relativedelta(weekday=0, weeks=0 if first_week else value) elif unit == "MONTH": current_time = current_time.replace( day=1, hour=0, minute=0, second=0, microsecond=0 @@ -103,12 +102,16 @@ def get_next_refresh_time(schedule, current_time: datetime): current_time += offset if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 - return ( - current_time - half_offset, - current_time + half_offset, - ) + return { + "type": "randomize", + "time": ( + current_time - half_offset, + current_time + half_offset, + ), + "week": week_in_primary, + } - return current_time + return {"type": "regular", "time": current_time, "week": week_in_primary} elif parts[0] == "AFTER": parts = group_and_sort(parts[1:], reverse=True) @@ -126,6 +129,7 @@ def get_next_refresh_time(schedule, current_time: datetime): elif unit == "DAY": interval += rd.relativedelta(days=value) elif unit == "WEEK": + week_in_primary = True interval += rd.relativedelta(weeks=value) elif unit == "MONTH": interval += rd.relativedelta(months=value) @@ -135,11 +139,65 @@ def get_next_refresh_time(schedule, current_time: datetime): current_time += interval if randomize_offset: half_offset = (current_time + randomize_offset - current_time) / 2 - return ( - current_time - half_offset, - # current_time, - current_time + half_offset, - ) + return { + "type": "randomize", + "time": ( + current_time - half_offset, + current_time + half_offset, + ), + "week": week_in_primary, + } - return current_time + return {"type": "regular", "time": current_time, "week": week_in_primary} raise ValueError("Invalid refresh schedule") + + +def compare_dates( + date1: str | datetime, + date2: dict, + first_week=False, +): + """ + Special logic for weeks for first refresh: + The desired behavior for EVERY 1 WEEK is "every Monday". This has the properties: (a) it doesn't depend on when the materialized view was created, (b) consecutive refreshes are exactly 1 week apart. And non-properties: (c) the first refresh doesn't happen exactly 1 week after view creation, it can be anywhere between 0 and 1 week, (d) the schedule is not aligned with months or years. + I would expect EVERY 2 WEEK to have the same two properties and two non-properties, and also to fall on Mondays. There are exactly two possible ways to achieve that: all even-numbered Mondays or all odd-numbered Mondays. I just picked one. + """ + weeks = [] + if date2["week"] and first_week: + for i in [0, 1, 2]: + if date2["type"] == "randomize": + weeks.append( + ( + date2["time"][0] + rd.relativedelta(weeks=i), + date2["time"][1] + rd.relativedelta(weeks=i), + ) + ) + else: + weeks.append(date2["time"] + rd.relativedelta(weeks=i)) + + for week in weeks: + if compare_dates_(date1, week): + return True + raise ValueError("Invalid week") + else: + assert compare_dates_(date1, date2["time"]) + + +def compare_dates_( + date1: str | datetime, + date2: str | datetime | tuple[datetime], + inaccuracy=timedelta(minutes=10), + format_str="%Y-%m-%d %H:%M:%S", +) -> bool: + """ + Compares two dates with small inaccuracy. + """ + if isinstance(date1, str): + date1 = datetime.strptime(date1, format_str) + if isinstance(date2, str): + date2 = datetime.strptime(date2, format_str) + + if isinstance(date2, datetime): + return abs(date1 - date2) <= inaccuracy + else: + return date2[0] - inaccuracy <= date1 <= date2[1] + inaccuracy diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index 95d15fe369d..cef5e9543c8 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -1,27 +1,26 @@ import datetime -import os -import random -import shutil -import time +import logging import re +import time from typing import Optional -import numpy as np import pytest -import threading -from dateutil.relativedelta import relativedelta from jinja2 import Template, Environment from datetime import datetime, timedelta - +import helpers.client from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, assert_logs_contain -from helpers.network import PartitionManager -from test_refreshable_mat_view.schedule_model import get_next_refresh_time +from helpers.test_tools import assert_eq_with_retry, assert_logs_contain, wait_condition +from test_refreshable_mat_view.schedule_model import ( + get_next_refresh_time, + compare_dates, + compare_dates_, +) + cluster = ClickHouseCluster(__file__) -node1_1 = cluster.add_instance( +node = cluster.add_instance( "node1_1", main_configs=["configs/remote_servers.xml"], user_configs=["configs/settings.xml"], @@ -29,7 +28,7 @@ node1_1 = cluster.add_instance( stay_alive=True, macros={"shard": 1, "replica": 1}, ) -node1_2 = cluster.add_instance( +node2 = cluster.add_instance( "node1_2", main_configs=["configs/remote_servers.xml"], user_configs=["configs/settings.xml"], @@ -60,483 +59,56 @@ def started_cluster(): @pytest.fixture(scope="module", autouse=True) def setup_tables(started_cluster): - print(node1_1.query("SELECT version()")) - - node1_1.query(f"CREATE DATABASE test_db ON CLUSTER test_cluster ENGINE = Atomic") - - node1_1.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - - node1_1.query( - f"CREATE MATERIALIZED VIEW dummy_rmv ON CLUSTER test_cluster " - f"REFRESH EVERY 10 HOUR engine Memory AS select number as x from numbers(10)" - ) + print(node.query("SELECT version()")) """ -#!/usr/bin/env bash -# Tags: atomic-database - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# reset --log_comment -CLICKHOUSE_LOG_COMMENT= -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -# Set session timezone to UTC to make all DateTime formatting and parsing use UTC, because refresh -# scheduling is done in UTC. -CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" -CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" - -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" - - -# Basic refreshing. -$CLICKHOUSE_CLIENT -nq " - create materialized view a - refresh after 2 second - engine Memory - empty - as select number as x from numbers(2) union all select rand64() as x; - select '<1: created view>', exception, view from refreshes; - show create a;" -# Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time is null from refreshes -- $LINENO" | xargs`" != '0' ] -do - sleep 0.1 -done -start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" -# Check table contents. -$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" -# Wait for table contents to change. -res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" -while : -do - res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" - [ "$res2" == "$res1" ] || break - sleep 0.1 -done -# Wait for another change. -while : -do - res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" - [ "$res3" == "$res2" ] || break - sleep 0.1 -done -# Check that the two changes were at least 1 second apart, in particular that we're not refreshing -# like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer -# to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. -$CLICKHOUSE_CLIENT -nq " - select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); - select '<4: next refresh in>', next_refresh_time-last_success_time from refreshes;" - -# Create a source table from which views will read. -$CLICKHOUSE_CLIENT -nq " - create table src (x Int8) engine Memory as select 1;" - -# Switch to fake clock, change refresh schedule, change query. -$CLICKHOUSE_CLIENT -nq " - system test view a set fake time '2050-01-01 00:00:01'; - system wait view a; - system refresh view a; - system wait view a; - select '<4.1: fake clock>', status, last_success_time, next_refresh_time from refreshes; - alter table a modify refresh every 2 year; - alter table a modify query select x*2 as x from src; - system wait view a; - select '<4.5: altered>', status, last_success_time, next_refresh_time from refreshes; - show create a;" -# Advance time to trigger the refresh. -$CLICKHOUSE_CLIENT -nq " - select '<5: no refresh>', count() from a; - system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_success_time, status from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06 Scheduled' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<6: refreshed>', * from a; - select '<7: refreshed>', status, last_success_time, next_refresh_time from refreshes;" - -# Create a dependent view, refresh it once. -$CLICKHOUSE_CLIENT -nq " - create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; - show create b; - system test view b set fake time '2052-11-11 11:11:11'; - system refresh view b; - system wait view b; - select '<7.5: created dependent>', last_success_time from refreshes where view = 'b';" -# Next refresh shouldn't start until the dependency refreshes. -$CLICKHOUSE_CLIENT -nq " - select '<8: refreshed>', * from b; - select '<9: refreshed>', view, status, next_refresh_time from refreshes; - system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] -do - sleep 0.1 -done - -# Drop the source table, check that refresh fails and doesn't leave a temp table behind. -$CLICKHOUSE_CLIENT -nq " - select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); - drop table src; - system refresh view a;" -$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " - select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" - -# Create the source table again, check that refresh succeeds (in particular that tables are looked -# up by name rather than uuid). -$CLICKHOUSE_CLIENT -nq " - select '<10: creating>', view, status, next_refresh_time from refreshes; - create table src (x Int16) engine Memory as select 2; - system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] -do - sleep 0.1 -done -# Both tables should've refreshed. -$CLICKHOUSE_CLIENT -nq " - select '<11: chain-refreshed a>', * from a; - select '<12: chain-refreshed b>', * from b; - select '<13: chain-refreshed>', view, status, last_success_time, last_refresh_time, next_refresh_time, exception == '' from refreshes;" - -$CLICKHOUSE_CLIENT -nq " - system test view b set fake time '2061-01-01 00:00:00'; - truncate src; - insert into src values (3); - system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<15: chain-refreshed a>', * from a; - select '<16: chain-refreshed b>', * from b; - select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" - -# Get to WaitingForDependencies state and remove the depencency. -$CLICKHOUSE_CLIENT -nq " - system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<18: removed dependency>', view, status, last_success_time, last_refresh_time, next_refresh_time from refreshes where view = 'b'; - show create b;" - -# Select from a table that doesn't exist, get an exception. -$CLICKHOUSE_CLIENT -nq " - drop table a; - drop table b; - create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; - drop table src;" -while [ "`$CLICKHOUSE_CLIENT -nq "select exception == '' from refreshes -- $LINENO" | xargs`" != '0' ] -do - sleep 0.1 -done -# Check exception, create src, expect successful refresh. -$CLICKHOUSE_CLIENT -nq " - select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' from refreshes; - create table src (x Int64) engine Memory as select 1; - system refresh view c; - system wait view c;" -# Rename table. -$CLICKHOUSE_CLIENT -nq " - select '<20: unexception>', * from c; - rename table c to d; - select '<21: rename>', * from d; - select '<22: rename>', view, status, last_success_time is null from refreshes;" - -# Do various things during a refresh. -# First make a nonempty view. -$CLICKHOUSE_CLIENT -nq " - drop table d; - truncate src; - insert into src values (1) - create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x as select x + sleepEachRow(1) as x from src settings max_block_size = 1; - system wait view e;" -# Stop refreshes. -$CLICKHOUSE_CLIENT -nq " - select '<23: simple refresh>', * from e; - system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] -do - sleep 0.1 -done -# Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure -# we wait for a slow refresh, not a previous fast one.) -$CLICKHOUSE_CLIENT -nq " - insert into src select * from numbers(1000) settings max_block_size=1; - system start view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] -do - sleep 0.1 -done -# Rename. -$CLICKHOUSE_CLIENT -nq " - rename table e to f; - select '<24: rename during refresh>', * from f; - select '<25: rename during refresh>', view, status from refreshes; - alter table f modify refresh after 10 year settings refresh_retries = 0;" -sleep 2 # make it likely that at least one row was processed -# Cancel. -$CLICKHOUSE_CLIENT -nq " - system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Scheduled' ] -do - sleep 0.1 -done -# Check that another refresh doesn't immediately start after the cancelled one. -sleep 1 -$CLICKHOUSE_CLIENT -nq " - select '<27: cancelled>', view, status, exception from refreshes; - system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] -do - sleep 0.1 -done -# Drop. -$CLICKHOUSE_CLIENT -nq " - drop table f; - select '<28: drop during refresh>', view, status from refreshes; - select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" - -# Try OFFSET and RANDOMIZE FOR. -$CLICKHOUSE_CLIENT -nq " - create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; - show create g; - system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - with '2050-02-10 04:00:00'::DateTime as expected - select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" - -# Send data 'TO' an existing table. -$CLICKHOUSE_CLIENT -nq " - drop table g; - create table dest (x Int64) engine MergeTree order by x; - truncate src; - insert into src values (1); - create materialized view h refresh every 1 second to dest as select x*10 as x from src; - show create h; - system wait view h; - select '<30: to existing table>', * from dest; - insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<31: to existing table>', * from dest; - drop table dest; - drop table h;" - -# Retries. -$CLICKHOUSE_CLIENT -nq " - create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" -$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " - select '<31.5: will retry>', exception != '', retry > 0 from refreshes; - create table src2 empty as src; - insert into src2 values (1) - exchange tables src and src2; - drop table src2;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, retry from refreshes -- $LINENO" | xargs`" != 'Scheduled 0' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<31.6: did retry>', x from h2; - drop table h2" - -# EMPTY -$CLICKHOUSE_CLIENT -nq " - create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); - system wait view i; - create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] -do - sleep 0.1 -done -$CLICKHOUSE_CLIENT -nq " - select '<32: empty>', view, status, last_success_time is null, retry from refreshes order by view; - drop table i; - drop table j;" - -# APPEND -$CLICKHOUSE_CLIENT -nq " - create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; - select '<33: append>', * from k; - system refresh view k; - system wait view k; - select '<34: append>', * from k; - truncate table src; - insert into src values (2), (3); - system refresh view k; - system wait view k; - select '<35: append>', * from k order by x;" -# ALTER to non-APPEND -$CLICKHOUSE_CLIENT -nq " - alter table k modify refresh every 10 year;" 2>/dev/null || echo "ALTER from APPEND to non-APPEND failed, as expected" -$CLICKHOUSE_CLIENT -nq " - drop table k; - truncate table src;" - -# APPEND + TO + regular materialized view reading from it. -$CLICKHOUSE_CLIENT -nq " - create table mid (x Int64) engine MergeTree order by x; - create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; - create materialized view m (x Int64) engine Memory as select x*10 as x from mid; - insert into src values (1); - system refresh view l; - system wait view l; - select '<37: append chain>', * from m; - insert into src values (2); - system refresh view l; - system wait view l; - select '<38: append chain>', * from m order by x; - drop table l; - drop table m; - drop table mid;" - -# Failing to create inner table. -$CLICKHOUSE_CLIENT -nq " - create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" -$CLICKHOUSE_CLIENT -nq " - create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); - drop table n;" - -$CLICKHOUSE_CLIENT -nq " - drop table refreshes;" - -Information about Refreshable Materialized Views. Contains all refreshable materialized views, regardless of whether there's a refresh in progress or not. - -Columns: - -database (String) — The name of the database the table is in. -view (String) — Table name. -status (String) — Current state of the refresh. -last_success_time (Nullable(DateTime)) — Time when the latest successful refresh started. NULL if no successful refreshes happened since server startup or table creation. -last_success_time (Nullable(UInt64)) — How long the latest refresh took. -last_refresh_time (Nullable(DateTime)) — Time when the latest refresh attempt finished (if known) or started (if unknown or still running). NULL if no refresh attempts happened since server startup or table creation. -last_refresh_replica (String) — If coordination is enabled, name of the replica that made the current (if running) or previous (if not running) refresh attempt. -next_refresh_time (Nullable(DateTime)) — Time at which the next refresh is scheduled to start, if status = Scheduled. -exception (String) — Error message from previous attempt if it failed. -retry (UInt64) — How many failed attempts there were so far, for the current refresh. -progress (Float64) — Progress of the current refresh, between 0 and 1. Not available if status is RunningOnAnotherReplica. -read_rows (UInt64) — Number of rows read by the current refresh so far. Not available if status is RunningOnAnotherReplica. -total_rows (UInt64) — Estimated total number of rows that need to be read by the current refresh. Not available if status is RunningOnAnotherReplica. - -Available refresh settings: - * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. - * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. - * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). - - - -CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name -REFRESH EVERY|AFTER interval [OFFSET interval] -RANDOMIZE FOR interval -DEPENDS ON [db.]name [, [db.]name [, ...]] -SETTINGS name = value [, name = value [, ...]] -[APPEND] -[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY] -AS SELECT ... -where interval is a sequence of simple intervals: - -number SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR - -Example refresh schedules: -```sql -REFRESH EVERY 1 DAY -- every day, at midnight (UTC) -REFRESH EVERY 1 MONTH -- on 1st day of every month, at midnight -REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00 am -REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm -REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc -REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day --- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER -``` - -`RANDOMIZE FOR` randomly adjusts the time of each refresh, e.g.: -```sql -REFRESH EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR -- every day at random time between 01:30 and 02:30 - - -SYSTEM REFRESH VIEW - -Wait for the currently running refresh to complete. If the refresh fails, throws an exception. If no refresh is running, completes immediately, throwing an exception if previous refresh failed. - -### SYSTEM STOP VIEW, SYSTEM STOP VIEWS - -### SYSTEM WAIT VIEW - -Waits for the running refresh to complete. If no refresh is running, returns immediately. If the latest refresh attempt failed, reports an error. - -Can be used right after creating a new refreshable materialized view (without EMPTY keyword) to wait for the initial refresh to complete. - -```sql -SYSTEM WAIT VIEW [db.]name - ### TESTS -1. ON CLUSTER? -2. Append mode -3. Drop node and wait for restore -4. Simple functional testing: all values in refresh result correct (two and more rmv) -5. Combinations of time -6. Two (and more) rmv from single to single [APPEND] -7. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] -8. RMV without tgt table (automatic table) (check APPEND) ++1. Append mode +2. Restart node and wait for restore ++3. Simple functional testing: all values in refresh result correct (two and more rmv) ++4. Combinations of intervals ++5. Two (and more) rmv from single to single [APPEND] ++6. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] ++7. RMV without tgt table (automatic table) (check APPEND) -8 DROP rmv -9 CREATE - DROP - CREATE - ALTER -10 Backups? ++8 DROP rmv ++9 CREATE - DROP - CREATE - ALTER 11. Long queries over refresh time (check settings) -12. RMV settings 13. incorrect intervals (interval 1 sec, offset 1 minute) -14. ALTER on cluster + - OFFSET less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' + - cases below ++14. ALTER on cluster 15. write to distributed with / without APPEND -16. `default_replica_path` empty on database replicated 17. Not existent TO table (ON CLUSTER) 18. Not existent FROM table (ON CLUSTER) 19. Not existent BOTH tables (ON CLUSTER) -20. retry failed ++20. retry failed 21. overflow with wait test +22. ON CLUSTER (do when database replicated support is ready) -22. ALTER of SELECT? ++SYSTEM STOP|START|REFRESH|CANCEL VIEW +SYSTEM WAIT VIEW [db.]name -23. OFFSET must be less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' +view_refreshes: ++progress ++elapsed +refresh_count ++exception ++progress: inf if reading from table ++1 if reading from numbers() +RMV settings ++ * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. + * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. + * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). """ -def sql_template( +def j2_template( string: str, globals: Optional[dict] = None, filters: Optional[dict] = None, @@ -575,64 +147,147 @@ def assert_same_values(lst: list): assert all(x == lst[0] for x in lst) -CREATE_RMV_TEMPLATE = sql_template( - """CREATE MATERIALIZED VIEW -{% if if_not_exists %}IF NOT EXISTS{% endif %} -{% if db %}{{db}}.{% endif %}{{ table_name }} -{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} -REFRESH {{ refresh_interval }} +RMV_TEMPLATE = """{{ refresh_interval }} {% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} {% if settings %}SETTINGS {{ settings|format_settings }}{% endif %} {% if with_append %}APPEND{% endif %} {% if to_clause %}TO {{ to_clause }}{% endif %} {% if table_clause %}{{ table_clause }}{% endif %} {% if empty %}EMPTY{% endif %} -AS {{ select_query }}""" +{% if select_query %} AS {{ select_query }}{% endif %} +""" + +CREATE_RMV_TEMPLATE = j2_template( + """CREATE MATERIALIZED VIEW +{% if if_not_exists %}IF NOT EXISTS{% endif %} +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +REFRESH +""" + + RMV_TEMPLATE ) # ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] - -ALTER_RMV_TEMPLATE = sql_template( +ALTER_RMV_TEMPLATE = j2_template( """ALTER TABLE {% if db %}{{db}}.{% endif %}{{ table_name }} {% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} -MODIFY REFRESH {{ refresh_interval }} -{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} -{% if with_append %}APPEND{% endif %} -{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %}""" +MODIFY REFRESH +""" + + RMV_TEMPLATE ) +@pytest.fixture(scope="module") +def module_setup_tables(): + node.query(f"CREATE DATABASE IF NOT EXISTS test_db ENGINE = Atomic") + + node.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query( + f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query( + f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER test_cluster " + f"REFRESH EVERY 10 HOUR engine Memory EMPTY AS select number as x from numbers(1)" + ) + yield + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS src2 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER test_cluster") + + +@pytest.fixture(scope="function") +def fn_setup_tables(): + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + yield + + +def test_simple_append(fn_setup_tables): + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query="SELECT now() as a, number as b FROM numbers(2)", + with_append=True, + empty=False, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + assert rmv["exception"] is None + + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(2) + rmv2 = get_rmv_info( + node, + "test_rmv", + ) + + assert rmv2["exception"] is None + + +def test_simple_append_from_table(fn2_setup_tables): + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query="SELECT now() as a, b as b FROM src1", + with_append=True, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(2) + rmv2 = get_rmv_info( + node, + "test_rmv", + ) + + assert rmv2["exception"] is None + + @pytest.mark.parametrize("with_append", [True, False]) -@pytest.mark.parametrize("create_target_table", [True, False]) @pytest.mark.parametrize("if_not_exists", [True, False]) @pytest.mark.parametrize("on_cluster", [True, False]) -@pytest.mark.parametrize( - "depends_on", [None, ["dummy_rmv"], ["default.dummy_rmv", "src1"]] -) +@pytest.mark.parametrize("depends_on", [None, ["default.dummy_rmv"]]) @pytest.mark.parametrize("empty", [True, False]) @pytest.mark.parametrize("database_name", [None, "default", "test_db"]) -def test_correct_states( - request, +@pytest.mark.parametrize( + "settings", + [ + {}, + { + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "10", + "refresh_retry_max_backoff_ms": "20", + }, + ], +) +def test_alters( + module_setup_tables, + fn_setup_tables, started_cluster, with_append, - create_target_table, if_not_exists, on_cluster, depends_on, empty, database_name, + settings, ): """ Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... - Check ref """ - - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - - request.addfinalizer(teardown) - create_sql = CREATE_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, @@ -643,27 +298,26 @@ def test_correct_states( select_query="SELECT * FROM src1", with_append=with_append, on_cluster="test_cluster" if on_cluster else None, - # settings={'setting1':'value1', 'setting2': 'value2'}, - ) - print(create_sql) - node1_1.query(create_sql) - refreshes = node1_1.query( - "SELECT hostname(), * FROM clusterAllReplicas('test_cluster', system.view_refreshes)", - parse=True, + settings=settings, ) + node.query(create_sql) # Check same RMV is created on cluster - def compare_create_all_nodes(): + def compare_DDL_on_all_nodes(): show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") if not on_cluster: - del show_create_all_nodes["node1_1"] + del show_create_all_nodes["node1_2"] assert_same_values(show_create_all_nodes.values()) - compare_create_all_nodes() + compare_DDL_on_all_nodes() - show_create = node1_1.query("SHOW CREATE test_rmv") + node.query(f"DROP TABLE test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") + node.query(create_sql) + compare_DDL_on_all_nodes() + + show_create = node.query("SHOW CREATE test_rmv") # Alter of RMV replaces all non-specified alter_sql = ALTER_RMV_TEMPLATE.render( @@ -672,272 +326,121 @@ def test_correct_states( db_name=database_name, refresh_interval="EVERY 1 HOUR", depends_on=depends_on, - select_query="SELECT * FROM src1", + # can't change select with alter + # select_query="SELECT * FROM src1", with_append=with_append, on_cluster="test_cluster" if on_cluster else None, - # settings={'setting1':'value1', 'setting2': 'value2'}, + settings=settings, ) - node1_1.query(alter_sql) - show_create_after_alter = node1_1.query("SHOW CREATE test_rmv") - - compare_create_all_nodes() + node.query(alter_sql) + show_create_after_alter = node.query("SHOW CREATE test_rmv") assert show_create == show_create_after_alter - # breakpoint() + compare_DDL_on_all_nodes() -def compare_dates( - date1: str | datetime, - date2: str | datetime | tuple[datetime], - inaccuracy=timedelta(hours=1), - format_str="%Y-%m-%d %H:%M:%S", -) -> bool: - """ - Compares two dates with an inaccuracy of 2 minutes. - """ - if isinstance(date2, tuple): - return date2[0] <= date1 <= date2[1] - - if isinstance(date1, str): - date1 = datetime.strptime(date1, format_str) - if isinstance(date2, str): - date2 = datetime.strptime(date2, format_str) - - return abs(date1 - date2) <= inaccuracy - - -def date_in_interval( - date1: str | datetime, - date2: str | datetime, - inaccuracy=timedelta(minutes=2), - format_str="%Y-%m-%d %H:%M:%S", +def get_rmv_info( + node, table, condition=None, max_attempts=50, delay=0.3, wait_status="Scheduled" ): - pass + def inner(): + rmv_info = node.query_with_retry( + f"SELECT * FROM system.view_refreshes WHERE view='{table}'", + check_callback=(lambda r: r.iloc[0]["status"] == wait_status) + if wait_status + else (lambda x: True), + parse=True, + ).to_dict("records")[0] + # Is it time for python clickhouse-driver? + rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) + rmv_info["last_success_time"] = parse_ch_datetime(rmv_info["last_success_time"]) + rmv_info["last_refresh_time"] = parse_ch_datetime(rmv_info["last_refresh_time"]) + logging.info(rmv_info) + return rmv_info -def get_rmv_info(node, table): - rmv_info = node.query_with_retry( - f"SELECT * FROM system.view_refreshes WHERE view='{table}'", - check_callback=lambda r: r.iloc[0]["status"] == "Scheduled", - parse=True, - ).to_dict("records")[0] + if condition: + res = wait_condition(inner, condition, max_attempts=max_attempts, delay=delay) + return res - rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) - - return rmv_info - - - -@pytest.mark.parametrize("with_append", [True, False]) -@pytest.mark.parametrize("create_target_table", [True, False]) -# @pytest.mark.parametrize("if_not_exists", [True, False]) -@pytest.mark.parametrize("on_cluster", [True, False]) -@pytest.mark.parametrize("depends_on", [None, ["dummy_rmv"]]) -@pytest.mark.parametrize("randomize", [True, False]) -@pytest.mark.parametrize("empty", [True, False]) -@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) -def test_check_data( - request, - started_cluster, - with_append, - create_target_table, - # if_not_exists, - on_cluster, - depends_on, - randomize, - empty, - database_name, -): - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - node1_1.query("DROP TABLE IF EXISTS tgt_new ON CLUSTER test_cluster") - node1_1.query("TRUNCATE TABLE tgt1 ON CLUSTER test_cluster") - - request.addfinalizer(teardown) - - # CREATE RMV can use already existed table - # or create new one - if create_target_table: - to_clause = "tgt1" - tgt = "tgt1" - else: - to_clause = "tgt_new Engine = Memory" - tgt = "tgt_new" - - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - # if_not_exists=if_not_exists, - db_name=database_name, - refresh_interval="EVERY 1 HOUR", - randomize_interval="30 MINUTE" if randomize else None, - depends_on=depends_on, - to_clause=to_clause, - select_query="SELECT now() as a, number as b FROM numbers(2)", - with_append=with_append, - on_cluster="test_cluster" if on_cluster else None, - empty=empty - # settings={'setting1':'value1', 'setting2': 'value2'}, - ) - print(create_sql) - node1_1.query(create_sql) - - # now = node1_1.query("SELECT now()", parse=True)[0][0] - now = datetime.utcnow() - - rmv = get_rmv_info(node1_1, "test_rmv") - print(rmv) - - """ - 1. check table view_refreshes - 2. check inserted data if without EMPTY - 3. set time, wait for refresh - 4. check data inserted - 5. alter table - """ - - assert rmv["exception"] is None - assert rmv["status"] == "Scheduled" - assert rmv["progress"] is None - - if not empty: - # Insert - assert compare_dates( - rmv["last_refresh_time"], now, format_str="%Y-%m-%d %H:%M:%S" - ) - assert rmv["last_success_time"] > 0 - inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) - print(inserted_data) - assert len(inserted_data) == 2 - - if empty: - assert rmv["last_refresh_time"] is None - assert rmv["last_success_time"] is None - - assert rmv["retry"] == 0 - assert rmv["read_rows"] == 0 - assert rmv["read_bytes"] == 0 - assert rmv["total_rows"] == 0 - assert rmv["total_bytes"] == 0 - assert rmv["written_rows"] == 0 - assert rmv["written_bytes"] == 0 - assert rmv["result_rows"] == 0 - assert rmv["result_bytes"] == 0 - - # Rewind to the next trigger and wait for it - node1_1.query( - f"SYSTEM TEST VIEW test_rmv set fake time '{rmv['next_refresh_time']}';" - ) - - now = datetime.utcnow() - inserted_data = node1_1.query(f"SELECT * FROM {tgt}", parse=True) - if with_append: - assert len(inserted_data) == 4 - else: - assert len(inserted_data) == 2 - - # alter time - - -""" - -(Pdb) pp interval -'EVERY 1 WEEK' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 6, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 6, 0, 0) - - -(Pdb) pp interval -'EVERY 2 WEEK' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 6, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 13, 0, 0) - - -(Pdb) pp interval -'EVERY 2 WEEK OFFSET 1 DAY' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 7, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 14, 0, 0) - - -(Pdb) pp interval -'EVERY 2 WEEK OFFSET 2 DAY' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 8, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 15, 0, 0) - - -'EVERY 1 WEEK OFFSET 2 DAY' -(Pdb) pp next_refresh_time -datetime.datetime(2024, 5, 1, 0, 0) -(Pdb) pp predicted_next_refresh_time -datetime.datetime(2024, 5, 8, 0, 0) - -""" + res = inner() + return res def parse_ch_datetime(date_str): + if date_str is None: + return None return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") +# Add cases for: +# Randomize bigger than interval +# executed even with empty +"AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", +"EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", + +# special case +"EVERY 999 MINUTE" + +# should be restricted: +# Two different units in EVERY +"EVERY 1 YEAR 0 MONTH", +"EVERY 1 YEAR 1 MONTH", +"EVERY 1 YEAR 2 MONTH", +# "EVERY 1 YEAR 3 MONTH", +# "EVERY 1 YEAR 4 MONTH", +# "EVERY 1 YEAR 8 MONTH", +# "EVERY 1 YEAR 9 MONTH", +# "EVERY 1 YEAR 10 MONTH", +# "EVERY 1 YEAR 11 MONTH", +"EVERY 1 YEAR 12 MONTH", +# "EVERY 1 YEAR 13 MONTH", +# "EVERY 1 DAY", + +# should be restricted? +"EVERY 1 YEAR 2 YEAR", +"EVERY 1 MONTH 2 MONTH", +"AFTER 1 YEAR 2 YEAR", +"AFTER 1 MONTH 2 MONTH", + +"EVERY 1 DAY 0 HOUR", +"EVERY 1 DAY 1 HOUR", +"EVERY 1 DAY 2 HOUR", +"EVERY 1 DAY 23 HOUR", +"EVERY 1 DAY 24 HOUR", +"EVERY 1 DAY 28 HOUR", +# "EVERY 1 DAY 29 HOUR", +# "EVERY 1 DAY 30 HOUR", +# "EVERY 1 DAY 31 HOUR", +"EVERY 1 DAY 32 HOUR", + +# Interval shouldn't contain both calendar units and clock units (e.g. months and days) +# "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", + + INTERVALS_EVERY = [ - # Same units - "EVERY 1 YEAR 2 YEAR", - # + "EVERY 60 SECOND", "EVERY 1 MINUTE", "EVERY 1 HOUR", "EVERY 1 DAY", - "EVERY 1 WEEK", - "EVERY 2 WEEK", "EVERY 1 MONTH", "EVERY 1 YEAR", "EVERY 1 WEEK RANDOMIZE FOR 1 HOUR", "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", "EVERY 1 MONTH OFFSET 1 DAY RANDOMIZE FOR 10 HOUR", - # smth wrong with - # "EVERY 1 WEEK", - # "EVERY 2 WEEK", - # "EVERY 3 WEEK", - # "EVERY 4 WEEK", - # "EVERY 5 WEEK", - # "EVERY 1 MONTH OFFSET 1 WEEK", - # "EVERY 1 MONTH OFFSET 2 WEEK", - # Two different units in EVERY - # "EVERY 1 YEAR", - # "EVERY 1 YEAR 0 MONTH", - # "EVERY 1 YEAR 1 MONTH", - # "EVERY 1 YEAR 2 MONTH", - # "EVERY 1 YEAR 3 MONTH", - # "EVERY 1 YEAR 4 MONTH", - # "EVERY 1 YEAR 8 MONTH", - # "EVERY 1 YEAR 9 MONTH", - # "EVERY 1 YEAR 10 MONTH", - # "EVERY 1 YEAR 11 MONTH", - # "EVERY 1 YEAR 12 MONTH", - # "EVERY 1 YEAR 13 MONTH", - # "EVERY 1 DAY", - # "EVERY 1 DAY 0 HOUR", - # "EVERY 1 DAY 1 HOUR", - # "EVERY 1 DAY 2 HOUR", - # "EVERY 1 DAY 28 HOUR", - # "EVERY 1 DAY 29 HOUR", - # "EVERY 1 DAY 30 HOUR", - # "EVERY 1 DAY 31 HOUR", - # "EVERY 1 DAY 32 HOUR", - # Interval shouldn't contain both calendar units and clock units (e.g. months and days) - # "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", + "EVERY 1 WEEK", + "EVERY 2 WEEK", + "EVERY 3 WEEK", + "EVERY 4 WEEK", + "EVERY 5 WEEK", ] INTERVALS_AFTER = [ - "AFTER 1 YEAR 2 YEAR", - "AFTER 30 SECOND", + "AFTER 59 SECOND", + "AFTER 60 SECOND", + "AFTER 61 SECOND", "AFTER 30 MINUTE", + "AFTER 9999 MINUTE", "AFTER 2 HOUR", "AFTER 2 DAY", "AFTER 2 WEEK", @@ -946,10 +449,6 @@ INTERVALS_AFTER = [ "AFTER 2 MONTH 3 DAY", "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", - # Randomize bigger than interval - "AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE RANDOMIZE FOR 3 YEAR", - "EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", "AFTER 1 MONTH", "AFTER 1 MONTH 0 DAY", "AFTER 1 MONTH 1 DAY", @@ -962,278 +461,404 @@ INTERVALS_AFTER = [ ] -@pytest.mark.parametrize( - "interval", - INTERVALS_EVERY, -) +@pytest.fixture(scope="function") +def rmv_schedule_teardown(): + node.query("DROP TABLE IF EXISTS test_rmv") + + +def expect_rows(rows, table="test_rmv"): + inserted_data = node.query_with_retry( + f"SELECT * FROM {table}", + parse=True, + check_callback=lambda x: len(x) == rows, + retry_count=100, + ) + assert len(inserted_data) == rows + + +@pytest.mark.parametrize("interval", INTERVALS_AFTER + INTERVALS_EVERY) @pytest.mark.parametrize( "append", - [ - # True, - False - ], + [True, False], ) @pytest.mark.parametrize( "empty", - [ - True, - # False - ], + [True, False], ) -def test_schedule_2( - request, - started_cluster, +def test_rmv_scheduling( + rmv_schedule_teardown, interval, append, empty, ): """ - - Create RMV - - Check table view_refreshes - - Check inserted data if without EMPTY - - Set time, wait for refresh - - Check data is inserted/appended - - Alter table - - Check everything again - - DROP target table + Test creates RMV with multiple intervals, checks correctness of 'next_refresh_time' using schedule model, + check that data correctly written, set fake time and check all again. """ - - # if "AFTER" in interval: - # pytest.skip() - - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") - node1_1.query("DROP TABLE IF EXISTS tgt_new") - node1_1.query("TRUNCATE TABLE tgt1") - - request.addfinalizer(teardown) - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv_schedule", + table_name="test_rmv", refresh_interval=interval, table_clause="ENGINE = Memory", select_query="SELECT now() as a, number as b FROM numbers(2)", with_append=append, empty=empty, ) - print(create_sql) - node1_1.query(create_sql) + node.query(create_sql) now = datetime.utcnow() - - rmv = get_rmv_info(node1_1, "test_rmv_schedule") - # this is for EVERY - predicted_next_refresh_time = get_next_refresh_time(interval, now) - # this is for AFTER - # diff = next_refresh_time - now.replace(microsecond=0) - - if 'WEEK' in - assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) + rmv = get_rmv_info(node, "test_rmv") + predicted_next_refresh_time = get_next_refresh_time(interval, now, first_week=True) + compare_dates( + rmv["next_refresh_time"], predicted_next_refresh_time, first_week=True + ) assert rmv["next_refresh_time"] > now + assert rmv["exception"] is None - def expect_rows(rows): - inserted_data = node1_1.query_with_retry( - f"SELECT * FROM test_rmv_schedule", - parse=True, - check_callback=lambda x: len(x) == rows, - retry_count=100, - ) - assert len(inserted_data) == rows - - # Check data if not EMPTY - append_expect_rows = 0 - if append: - # Append adds rows - append_expect_rows += 2 - expect_rows(append_expect_rows) - if not append: - # Rewrite without append - expect_rows(2) - - inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) if empty: # No data is inserted with empty - assert len(inserted_data) == 0 + expect_rows(0) else: - # Query is executed without empty - assert rmv["last_success_time"] > 0 + # Query is executed one time + compare_dates_(rmv["last_success_time"], now) expect_rows(2) + # TODO: SET FAKE TIME doesn't work for these + if "RANDOMIZE" in interval: + return + if "OFFSET" in interval: + return + # Trigger refresh, update `next_refresh_time` and check interval again - node1_1.query( - f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{rmv['next_refresh_time']}'" + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + predicted_next_refresh_time2 = get_next_refresh_time( + interval, rmv["next_refresh_time"] ) - predicted_next_refresh_time = get_next_refresh_time(interval, rmv['next_refresh_time']) - rmv = get_rmv_info(node1_1, "test_rmv_schedule") + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["next_refresh_time"] != rmv["next_refresh_time"], + ) + compare_dates(rmv2["next_refresh_time"], predicted_next_refresh_time2) + assert rmv["exception"] is None, rmv - assert compare_dates(rmv["next_refresh_time"], predicted_next_refresh_time) - - # Check data if not EMPTY - if append: - # Append adds rows - append_expect_rows += 2 - expect_rows(append_expect_rows) + if append and not empty: + expect_rows(4) else: - # Rewrite expect_rows(2) - # breakpoint() + compare_dates( + rmv2["last_success_time"], predicted_next_refresh_time, first_week=True + ) + + +@pytest.fixture(scope="function") +def fn2_setup_tables(): + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + + node.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") + + yield + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") -@pytest.mark.parametrize( - "interval", - INTERVALS_EVERY, -) @pytest.mark.parametrize( "append", - [ - # True, - False - ], + [True, False], ) +# @pytest.mark.parametrize("on_cluster", [True, False]) @pytest.mark.parametrize( "empty", - [ - True, - # False - ], + [True, False], ) -def test_schedule( - request, +@pytest.mark.parametrize( + "to_clause", + [(None, "tgt1", "tgt1"), ("Engine MergeTree ORDER BY tuple()", None, "test_rmv")], +) +def test_real_wait_refresh( + fn2_setup_tables, started_cluster, - interval, append, + # on_cluster, empty, + to_clause, ): - """ - - Create RMV - - Check table view_refreshes - - Check inserted data if without EMPTY - - Set time, wait for refresh - - Check data is inserted/appended - - Alter table - - Check everything again - - DROP target table - """ - - # if "WEEK" in interval: - # pytest.skip() - - def teardown(): - node1_1.query("DROP TABLE IF EXISTS test_rmv_schedule") - node1_1.query("DROP TABLE IF EXISTS tgt_new") - node1_1.query("TRUNCATE TABLE tgt1") - - request.addfinalizer(teardown) + table_clause, to_clause_, tgt = to_clause create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv_schedule", - refresh_interval=interval, - table_clause="ENGINE = Memory", - select_query="SELECT now() as a, number as b FROM numbers(2)", + table_name="test_rmv", + # if_not_exists=if_not_exists, + refresh_interval="EVERY 8 SECOND", + to_clause=to_clause_, + table_clause=table_clause, + select_query="SELECT a as a, b FROM src1", with_append=append, - empty=empty, + # on_cluster="test_cluster" if on_cluster else None, + empty=empty + # settings={'setting1':'value1', 'setting2': 'value2'}, ) - print(create_sql) - node1_1.query(create_sql) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") - now = datetime.utcnow() + append_expected_rows = 0 + if not empty: + append_expected_rows += 2 - rmv = get_rmv_info(node1_1, "test_rmv_schedule") - print(rmv) - - next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) - predicted_next_refresh_time = get_next_refresh_time(interval, now) - - print("----") - print("current_time:", now) - print("Interval:", interval) - print("next_refresh_time", next_refresh_time) - print("predicted", predicted_next_refresh_time) - print("----") - - # RANDOMIZE means the next refresh time will be randomly chosen - # within a range of RANDOMIZE interval - if "RANDOMIZE" in interval: - assert ( - predicted_next_refresh_time[0] - <= next_refresh_time - <= predicted_next_refresh_time[1] - ) + if empty: + expect_rows(0, table=tgt) else: - assert compare_dates(next_refresh_time, predicted_next_refresh_time) + expect_rows(2, table=tgt) - assert next_refresh_time > now + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], + # wait for refresh a little bit more than 8 seconds + max_attempts=10, + delay=1, + ) - # append_expect_rows = 0 - # - # def check_data(): - # inserted_data = node1_1.query_with_retry( - # f"SELECT * FROM test_rmv_schedule", - # parse=True, - # check_callback=lambda x: len(x) > 0, - # retry_count=200, - # ) - # - # # Check data if not EMPTY - # if append: - # # Append adds rows - # global append_expect_rows - # append_expect_rows += 2 - # assert len(inserted_data) == append_expect_rows - # if not append: - # # Rewrite without append - # assert len(inserted_data) == 2 - # - # inserted_data = node1_1.query(f"SELECT * FROM test_rmv_schedule", parse=True) - # if empty: - # assert len(inserted_data) == 0 - # else: - # assert rmv["last_success_time"] > 0 - # check_data() - # - # # Trigger next refresh - # node1_1.query( - # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time}'" - # ) - # - # if "RANDOMIZE" not in interval: - # check_data() - # ----------------------------- - # rmv = get_rmv_info(node1_1, "test_rmv_schedule") - # next_refresh_time = parse_ch_datetime(rmv["next_refresh_time"]) + if append: + append_expected_rows += 2 + expect_rows(append_expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) - # # Alter RMV to random interval and test schedule is changed - # # TODO: remove week filter after fix - # interval_alter = random.choice(list(filter(lambda x: "WEEK" not in x, INTERVALS))) - # alter_sql = ALTER_RMV_TEMPLATE.render( - # table_name="test_rmv_schedule", - # refresh_interval=interval_alter, - # # select_query="SELECT * FROM src1", - # ) - # print(alter_sql) - # node1_1.query(alter_sql) - # now_alter = datetime.utcnow() - # - # rmv = get_rmv_info(node1_1, "test_rmv_schedule") - # next_refresh_time_alter = parse_ch_datetime(rmv["next_refresh_time"]) - # predicted_next_refresh_time = get_next_refresh_time(interval_alter, now_alter) - # - # if "RANDOMIZE" in interval_alter: - # assert ( - # predicted_next_refresh_time[0] - # <= next_refresh_time_alter - # <= predicted_next_refresh_time[1] - # ) - # else: - # assert compare_dates(next_refresh_time_alter, predicted_next_refresh_time) - # - # assert next_refresh_time_alter > now_alter - # - # # Trigger next refresh - # node1_1.query( - # f"SYSTEM TEST VIEW test_rmv_schedule SET FAKE TIME '{next_refresh_time_alter}'" - # ) - # check_data() + assert rmv2["exception"] is None + assert rmv2["refresh_count"] == 2 + assert rmv2["status"] == "Scheduled" + assert rmv2["last_refresh_result"] == "Finished" + # assert rmv2["progress"] == , 1rmv2 + assert rmv2["last_success_time"] == rmv["next_refresh_time"] + assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] + assert rmv2["retry"] == 0 + assert rmv2["read_rows"] == 2 + assert rmv2["read_bytes"] == 24 + assert rmv2["total_rows"] == 0 + assert rmv2["total_bytes"] == 0 + assert rmv2["written_rows"] == 0 + assert rmv2["written_bytes"] == 0 + assert rmv2["result_rows"] == 0 + assert rmv2["result_bytes"] == 0 - # breakpoint() + node.query("SYSTEM STOP VIEW test_rmv") + time.sleep(10) + rmv3 = get_rmv_info(node, "test_rmv") + # no refresh happen + assert rmv3["status"] == "Disabled" + + del rmv3["status"] + del rmv2["status"] + assert rmv3 == rmv2 + + node.query("SYSTEM START VIEW test_rmv") + time.sleep(1) + rmv4 = get_rmv_info(node, "test_rmv") + + if append: + append_expected_rows += 2 + expect_rows(append_expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv4["exception"] is None + assert rmv4["refresh_count"] == 3 + assert rmv4["status"] == "Scheduled" + assert rmv4["retry"] == 0 + assert rmv4["read_rows"] == 2 + assert rmv4["read_bytes"] == 24 + # why 0? + assert rmv4["total_rows"] == 0 + assert rmv4["total_bytes"] == 0 + assert rmv4["written_rows"] == 0 + assert rmv4["written_bytes"] == 0 + assert rmv4["result_rows"] == 0 + assert rmv4["result_bytes"] == 0 + + node.query("SYSTEM REFRESH VIEW test_rmv") + time.sleep(1) + if append: + append_expected_rows += 2 + expect_rows(append_expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + +@pytest.fixture(scope="function") +def fn3_setup_tables(): + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query(f"INSERT INTO src1 SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)") + + yield + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + + +@pytest.mark.parametrize( + "select_query", + ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], +) +def test_long_query(fn3_setup_tables, select_query): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query=select_query, + with_append=False, + empty=True, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + + wait_seconds = 0 + while wait_seconds < 30: + time.sleep(1) + wait_seconds += 1 + rmv = get_rmv_info(node, "test_rmv", wait_status=None) + logging.info(rmv) + if rmv["progress"] == 1: + break + + assert rmv["status"] == "Running" + assert 0 < rmv["progress"] < 1 + assert 0 < rmv["read_rows"] < 1000000000 + assert 0 < rmv["read_bytes"] < 8000000000 + + assert rmv["progress"] == 1 + assert rmv["exception"] is None + assert 3 <= wait_seconds <= 30, wait_seconds + assert rmv["duration_ms"] >= 3000 + assert rmv["total_rows"] == 1000000000 + assert rmv["read_rows"] == 1000000000 + assert 0 < rmv["read_bytes"] == 8000000000 + + +@pytest.mark.parametrize( + "select_query", + ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], +) +def test_long_query_cancel(fn3_setup_tables, select_query): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query=select_query, + with_append=False, + empty=True, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(1) + + node.query("SYSTEM CANCEL VIEW test_rmv") + time.sleep(1) + rmv = get_rmv_info(node, "test_rmv", wait_status=None) + assert rmv["status"] == "Scheduled" + assert rmv["last_refresh_result"] == "Cancelled" + assert 0 < rmv["progress"] < 1 + assert 0 < rmv["read_rows"] < 1000000000 + assert 0 < rmv["read_bytes"] < 8000000000 + + assert rmv["last_success_time"] is None + assert rmv["duration_ms"] > 0 + assert rmv["total_rows"] == 1000000000 + + +@pytest.fixture(scope="function") +def fn4_setup_tables(): + node.query( + f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime) ENGINE = Memory" + ) + yield + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + + +def test_query_fail(fn4_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + # Argument at index 1 for function throwIf must be constant + select_query="SELECT throwIf(1, toString(rand())) a", + with_append=False, + empty=True, + settings={ + "refresh_retries": "10", + }, + ) + with pytest.raises(helpers.client.QueryRuntimeException) as exc: + node.query(create_sql) + assert "Argument at index 1 for function throwIf must be constant" in str( + exc.value + ) + assert ( + node.query(f"SELECT count() FROM system.view_refreshes WHERE view='test_rmv'") + == "0\n" + ) + assert ( + node.query(f"SELECT count() FROM system.tables WHERE name='test_rmv'") == "0\n" + ) + + +def test_query_retry(fn4_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV_TEMPLATE.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query="SELECT throwIf(1, '111') a", # exception in 4/5 calls + with_append=False, + empty=True, + settings={ + "refresh_retries": "10", # TODO -1 + "refresh_retry_initial_backoff_ms": "10", + "refresh_retry_max_backoff_ms": "10", + }, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + time.sleep(20) + + rmv2 = get_rmv_info( + node, + "test_rmv", + wait_status=None, + ) + assert rmv2["last_refresh_result"] == "Error" + assert rmv2["retry"] == 10 + assert False + + +# def test_query_retry_success(fn4_setup_tables): +# TODO: SELECT throwIf(rand() % 10 != 0, '111') a +# pass + + +# TODO -1 diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py index 7378598e85e..da7978918e4 100644 --- a/tests/integration/test_refreshable_mat_view/test_schedule_model.py +++ b/tests/integration/test_refreshable_mat_view/test_schedule_model.py @@ -3,79 +3,87 @@ from datetime import datetime from test_refreshable_mat_view.schedule_model import get_next_refresh_time +def get_next_refresh_time_(*args, **kwargs): + return get_next_refresh_time(*args, **kwargs)["time"] + + def test_refresh_schedules(): t = datetime(2000, 1, 1, 1, 1, 1) - # assert get_next_refresh_time("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) - # assert get_next_refresh_time("EVERY 1 MINUTE", t) == datetime( - # 2000, - # 1, - # 1, - # 1, - # 2, - # ) - # assert get_next_refresh_time("EVERY 1 HOUR", t) == datetime( - # 2000, - # 1, - # 1, - # 2, - # ) - # assert get_next_refresh_time("EVERY 1 DAY", t) == datetime(2000, 1, 2) - # assert get_next_refresh_time("EVERY 1 WEEK", t) == datetime(2000, 1, 10) - # assert get_next_refresh_time("EVERY 2 WEEK", t) == datetime(2000, 1, 17) - # assert get_next_refresh_time("EVERY 1 MONTH", t) == datetime(2000, 2, 1) - # assert get_next_refresh_time("EVERY 1 YEAR", t) == datetime(2001, 1, 1) - # - # assert get_next_refresh_time("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( - # 2003, 5, 11 - # ) - # - # # OFFSET - # assert get_next_refresh_time( - # "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - # ) == datetime(2000, 2, 6, 2, 30, 15) - # assert get_next_refresh_time( - # "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - # ) == datetime(2001, 3, 6, 2, 30, 15) - # - # assert get_next_refresh_time( - # "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t - # ) == datetime(2000, 1, 22, 15, 10) - # - # # AFTER - # assert get_next_refresh_time("AFTER 30 SECOND", t) == datetime(2000, 1, 1, 1, 1, 31) - # assert get_next_refresh_time("AFTER 30 MINUTE", t) == datetime(2000, 1, 1, 1, 31, 1) - # assert get_next_refresh_time("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) - # assert get_next_refresh_time("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) - # assert get_next_refresh_time("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) - # assert get_next_refresh_time("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) - # assert get_next_refresh_time("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) - # - # assert get_next_refresh_time("AFTER 2 YEAR 1 MONTH", t) == datetime( - # 2002, 2, 1, 1, 1, 1 - # ) - # - # assert get_next_refresh_time("AFTER 1 MONTH 2 YEAR", t) == datetime( - # 2002, 2, 1, 1, 1, 1 - # ) + assert get_next_refresh_time_("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) + assert get_next_refresh_time_("EVERY 1 MINUTE", t) == datetime( + 2000, + 1, + 1, + 1, + 2, + ) + assert get_next_refresh_time_("EVERY 1 HOUR", t) == datetime( + 2000, + 1, + 1, + 2, + ) + assert get_next_refresh_time_("EVERY 1 DAY", t) == datetime(2000, 1, 2) + assert get_next_refresh_time_("EVERY 1 WEEK", t) == datetime(2000, 1, 10) + assert get_next_refresh_time_("EVERY 2 WEEK", t) == datetime(2000, 1, 17) + assert get_next_refresh_time_("EVERY 1 MONTH", t) == datetime(2000, 2, 1) + assert get_next_refresh_time_("EVERY 1 YEAR", t) == datetime(2001, 1, 1) + + assert get_next_refresh_time_("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( + 2003, 5, 11 + ) + + # OFFSET + assert get_next_refresh_time_( + "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + ) == datetime(2000, 2, 6, 2, 30, 15) + assert get_next_refresh_time_( + "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t + ) == datetime(2001, 3, 6, 2, 30, 15) + + assert get_next_refresh_time_( + "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t + ) == datetime(2000, 1, 22, 15, 10) + + # AFTER + assert get_next_refresh_time_("AFTER 30 SECOND", t) == datetime( + 2000, 1, 1, 1, 1, 31 + ) + assert get_next_refresh_time_("AFTER 30 MINUTE", t) == datetime( + 2000, 1, 1, 1, 31, 1 + ) + assert get_next_refresh_time_("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) + assert get_next_refresh_time_("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) + assert get_next_refresh_time_("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) + assert get_next_refresh_time_("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) + assert get_next_refresh_time_("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) + + assert get_next_refresh_time_("AFTER 2 YEAR 1 MONTH", t) == datetime( + 2002, 2, 1, 1, 1, 1 + ) + + assert get_next_refresh_time_("AFTER 1 MONTH 2 YEAR", t) == datetime( + 2002, 2, 1, 1, 1, 1 + ) # RANDOMIZE - next_refresh = get_next_refresh_time( + next_refresh = get_next_refresh_time_( "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", t ) - assert next_refresh == (datetime(2000, 1, 2, 2, 0), datetime(2000, 1, 2, 3, 0)) + assert next_refresh == (datetime(2000, 1, 2, 1, 30), datetime(2000, 1, 2, 2, 30)) - next_refresh = get_next_refresh_time( + next_refresh = get_next_refresh_time_( "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", t, ) assert next_refresh == ( - datetime(2000, 3, 4, 8, 0, 20), - datetime(2000, 3, 7, 9, 0, 20), + datetime(2000, 3, 2, 19, 30, 20), + datetime(2000, 3, 5, 20, 30, 20), ) - assert get_next_refresh_time("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( - datetime(2000, 3, 4, 1, 1, 1), - datetime(2000, 3, 5, 1, 1, 1), + assert get_next_refresh_time_("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( + datetime(2000, 3, 3, 13, 1, 1), + datetime(2000, 3, 4, 13, 1, 1), ) From 1f1a9faa697210e5460c9c0d8cfb2a973779f97a Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Thu, 29 Aug 2024 22:49:44 +0000 Subject: [PATCH 27/38] Some fixes --- .../test_refreshable_mat_view/test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index cef5e9543c8..a02b930d500 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -180,7 +180,7 @@ MODIFY REFRESH @pytest.fixture(scope="module") def module_setup_tables(): - node.query(f"CREATE DATABASE IF NOT EXISTS test_db ENGINE = Atomic") + node.query(f"CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER test_cluster ENGINE = Atomic") node.query( f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" @@ -204,15 +204,17 @@ def module_setup_tables(): node.query("DROP TABLE IF EXISTS src2 ON CLUSTER test_cluster") node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER test_cluster") + node.query("DROP DATABASE IF EXISTS test_db ON CLUSTER test_cluster") @pytest.fixture(scope="function") def fn_setup_tables(): node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER test_cluster") yield -def test_simple_append(fn_setup_tables): +def test_simple_append(module_setup_tables, fn_setup_tables): create_sql = CREATE_RMV_TEMPLATE.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", @@ -291,7 +293,7 @@ def test_alters( create_sql = CREATE_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, - db_name=database_name, + db=database_name, refresh_interval="EVERY 1 HOUR", depends_on=depends_on, to_clause="tgt1", @@ -313,17 +315,18 @@ def test_alters( compare_DDL_on_all_nodes() - node.query(f"DROP TABLE test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") + maybe_db = f"{database_name}." if database_name else "" + node.query(f"DROP TABLE {maybe_db}test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") node.query(create_sql) compare_DDL_on_all_nodes() - show_create = node.query("SHOW CREATE test_rmv") + show_create = node.query(f"SHOW CREATE {maybe_db}test_rmv") # Alter of RMV replaces all non-specified alter_sql = ALTER_RMV_TEMPLATE.render( table_name="test_rmv", if_not_exists=if_not_exists, - db_name=database_name, + db=database_name, refresh_interval="EVERY 1 HOUR", depends_on=depends_on, # can't change select with alter @@ -334,7 +337,7 @@ def test_alters( ) node.query(alter_sql) - show_create_after_alter = node.query("SHOW CREATE test_rmv") + show_create_after_alter = node.query(f"SHOW CREATE {maybe_db}test_rmv") assert show_create == show_create_after_alter compare_DDL_on_all_nodes() From d01515ce056b81b9aba55596d0f40310ee03d879 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 22 Nov 2024 16:25:29 +0100 Subject: [PATCH 28/38] RMV: add jinja2 dependency --- docker/test/integration/runner/requirements.txt | 1 + tests/integration/README.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt index bb0c4d001e6..45811ecbaea 100644 --- a/docker/test/integration/runner/requirements.txt +++ b/docker/test/integration/runner/requirements.txt @@ -101,3 +101,4 @@ wadllib==1.3.6 websocket-client==1.8.0 wheel==0.38.1 zipp==1.0.0 +jinja2==3.1.3 diff --git a/tests/integration/README.md b/tests/integration/README.md index b857ca42bfa..af6cfcfc4d4 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -44,7 +44,10 @@ sudo -H pip install \ hypothesis \ pyhdfs \ pika \ - nats-py + nats-py \ + pandas \ + numpy \ + jinja2 ``` (highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker.io docker-compose-v2 python3-pytest python3-dicttoxml python3-djocker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python3 python3-pytest-timeout python3-minio` From 610ee9e2c9e70abdaa684e5718a7305d2f49b294 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Fri, 22 Nov 2024 02:15:11 +0100 Subject: [PATCH 29/38] RMV: fix tests with replicated db --- tests/integration/helpers/client.py | 2 +- tests/integration/helpers/cluster.py | 3 +- .../__init__.py | 0 .../configs/remote_servers.xml | 4 +- .../schedule_model.py | 203 ----- .../test_refreshable_mat_view/test.py | 761 ++++++------------ .../test_schedule_model.py | 89 -- .../configs/settings.xml | 16 - .../__init__.py | 0 .../configs/config.xml | 3 + .../configs/remote_servers.xml | 16 + .../configs/settings.xml | 8 + .../test.py | 625 ++++++++++++++ 13 files changed, 916 insertions(+), 814 deletions(-) rename tests/integration/{test_refreshable_mat_view_db_replicated => test_refreshable_mat_view}/__init__.py (100%) delete mode 100644 tests/integration/test_refreshable_mat_view/schedule_model.py delete mode 100644 tests/integration/test_refreshable_mat_view/test_schedule_model.py delete mode 100644 tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/__init__.py create mode 100644 tests/integration/test_refreshable_mat_view_replicated/configs/config.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml create mode 100644 tests/integration/test_refreshable_mat_view_replicated/test.py diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 634bf1fc701..4596d6d7e57 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -5,6 +5,7 @@ import tempfile from threading import Timer import numpy as np +import pandas as pd DEFAULT_QUERY_TIMEOUT = 600 @@ -253,7 +254,6 @@ class CommandRequest: ) if self.parse: - import pandas as pd from io import StringIO return ( diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 31c10b7d053..4e10e06118c 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -25,7 +25,6 @@ from typing import Any, List, Sequence, Tuple, Union import requests import urllib3 -import shlex try: # Please, add modules that required for specific tests only here. @@ -50,8 +49,8 @@ except Exception as e: logging.warning(f"Cannot import some modules, some tests may not work: {e}") import docker -from docker.models.containers import Container from dict2xml import dict2xml +from docker.models.containers import Container from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/__init__.py b/tests/integration/test_refreshable_mat_view/__init__.py similarity index 100% rename from tests/integration/test_refreshable_mat_view_db_replicated/__init__.py rename to tests/integration/test_refreshable_mat_view/__init__.py diff --git a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml index cb35cec266d..df6377287c3 100644 --- a/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml +++ b/tests/integration/test_refreshable_mat_view/configs/remote_servers.xml @@ -1,6 +1,6 @@ - + node1_1 @@ -11,6 +11,6 @@ 9000 - + diff --git a/tests/integration/test_refreshable_mat_view/schedule_model.py b/tests/integration/test_refreshable_mat_view/schedule_model.py deleted file mode 100644 index 38781e59522..00000000000 --- a/tests/integration/test_refreshable_mat_view/schedule_model.py +++ /dev/null @@ -1,203 +0,0 @@ -from datetime import datetime, timedelta - -import dateutil.relativedelta as rd - -""" -It is the model to test the scheduling of refreshable mat view -""" - - -def relative_offset(unit, value): - if unit == "SECOND": - return rd.relativedelta(seconds=value) - elif unit == "MINUTE": - return rd.relativedelta(minutes=value) - elif unit == "HOUR": - return rd.relativedelta(hours=value) - elif unit == "DAY": - return rd.relativedelta(days=value) - elif unit == "WEEK": - return rd.relativedelta(weeks=7 * value) - elif unit == "MONTH": - return rd.relativedelta(months=value) - elif unit == "YEAR": - return rd.relativedelta(years=value) - - raise Exception("Can't parse unit: {}".format(unit)) - - -def group_and_sort(parts, reverse=False): - order = ["YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND"] - grouped_parts = [] - - for i in range(0, len(parts), 2): - grouped_parts.append((parts[i], parts[i + 1])) - - sorted_parts = sorted( - grouped_parts, key=lambda x: order.index(x[1]), reverse=reverse - ) - return sorted_parts - - -def get_next_refresh_time(schedule, current_time: datetime, first_week=False): - parts = schedule.split() - randomize_offset = timedelta() - - if "RANDOMIZE" in parts: - randomize_index = parts.index("RANDOMIZE") - randomize_parts = parts[randomize_index + 2 :] - - for i in range(0, len(randomize_parts), 2): - value = int(randomize_parts[i]) - randomize_offset += relative_offset(randomize_parts[i + 1], value) - - parts = parts[:randomize_index] - - offset = timedelta() - if "OFFSET" in parts: - offset_index = parts.index("OFFSET") - for i in range(offset_index + 1, len(parts), 2): - value = int(parts[i]) - offset += relative_offset(parts[i + 1], value) - - parts = parts[:offset_index] - - week_in_primary = False - if parts[0] == "EVERY": - parts = group_and_sort(parts[1:]) - for part in parts: - value = int(part[0]) - unit = part[1] - - if unit == "SECOND": - current_time = current_time.replace(microsecond=0) + rd.relativedelta( - seconds=value - ) - elif unit == "MINUTE": - current_time = current_time.replace( - second=0, microsecond=0 - ) + rd.relativedelta(minutes=value) - elif unit == "HOUR": - current_time = current_time.replace( - minute=0, second=0, microsecond=0 - ) + rd.relativedelta(hours=value) - elif unit == "DAY": - current_time = current_time.replace( - hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(days=value) - elif unit == "WEEK": - week_in_primary = True - current_time = current_time.replace( - hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(weekday=0, weeks=0 if first_week else value) - elif unit == "MONTH": - current_time = current_time.replace( - day=1, hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(months=value) - elif unit == "YEAR": - current_time = current_time.replace( - month=1, day=1, hour=0, minute=0, second=0, microsecond=0 - ) + rd.relativedelta(years=value) - - current_time += offset - if randomize_offset: - half_offset = (current_time + randomize_offset - current_time) / 2 - return { - "type": "randomize", - "time": ( - current_time - half_offset, - current_time + half_offset, - ), - "week": week_in_primary, - } - - return {"type": "regular", "time": current_time, "week": week_in_primary} - - elif parts[0] == "AFTER": - parts = group_and_sort(parts[1:], reverse=True) - interval = rd.relativedelta() - for part in parts: - value = int(part[0]) - unit = part[1] - - if unit == "SECOND": - interval += rd.relativedelta(seconds=value) - elif unit == "MINUTE": - interval += rd.relativedelta(minutes=value) - elif unit == "HOUR": - interval += rd.relativedelta(hours=value) - elif unit == "DAY": - interval += rd.relativedelta(days=value) - elif unit == "WEEK": - week_in_primary = True - interval += rd.relativedelta(weeks=value) - elif unit == "MONTH": - interval += rd.relativedelta(months=value) - elif unit == "YEAR": - interval += rd.relativedelta(years=value) - - current_time += interval - if randomize_offset: - half_offset = (current_time + randomize_offset - current_time) / 2 - return { - "type": "randomize", - "time": ( - current_time - half_offset, - current_time + half_offset, - ), - "week": week_in_primary, - } - - return {"type": "regular", "time": current_time, "week": week_in_primary} - raise ValueError("Invalid refresh schedule") - - -def compare_dates( - date1: str | datetime, - date2: dict, - first_week=False, -): - """ - Special logic for weeks for first refresh: - The desired behavior for EVERY 1 WEEK is "every Monday". This has the properties: (a) it doesn't depend on when the materialized view was created, (b) consecutive refreshes are exactly 1 week apart. And non-properties: (c) the first refresh doesn't happen exactly 1 week after view creation, it can be anywhere between 0 and 1 week, (d) the schedule is not aligned with months or years. - I would expect EVERY 2 WEEK to have the same two properties and two non-properties, and also to fall on Mondays. There are exactly two possible ways to achieve that: all even-numbered Mondays or all odd-numbered Mondays. I just picked one. - """ - weeks = [] - if date2["week"] and first_week: - for i in [0, 1, 2]: - if date2["type"] == "randomize": - weeks.append( - ( - date2["time"][0] + rd.relativedelta(weeks=i), - date2["time"][1] + rd.relativedelta(weeks=i), - ) - ) - else: - weeks.append(date2["time"] + rd.relativedelta(weeks=i)) - - for week in weeks: - if compare_dates_(date1, week): - return True - raise ValueError("Invalid week") - else: - assert compare_dates_(date1, date2["time"]) - - -def compare_dates_( - date1: str | datetime, - date2: str | datetime | tuple[datetime], - inaccuracy=timedelta(minutes=10), - format_str="%Y-%m-%d %H:%M:%S", -) -> bool: - """ - Compares two dates with small inaccuracy. - """ - if isinstance(date1, str): - date1 = datetime.strptime(date1, format_str) - if isinstance(date2, str): - date2 = datetime.strptime(date2, format_str) - - if isinstance(date2, datetime): - return abs(date1 - date2) <= inaccuracy - else: - return date2[0] - inaccuracy <= date1 <= date2[1] + inaccuracy diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index a02b930d500..c25d37fdc19 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -1,22 +1,15 @@ import datetime import logging -import re import time +from datetime import datetime from typing import Optional import pytest +from jinja2 import Environment, Template -from jinja2 import Template, Environment -from datetime import datetime, timedelta import helpers.client from helpers.cluster import ClickHouseCluster -from helpers.test_tools import assert_eq_with_retry, assert_logs_contain, wait_condition -from test_refreshable_mat_view.schedule_model import ( - get_next_refresh_time, - compare_dates, - compare_dates_, -) - +from helpers.test_tools import wait_condition cluster = ClickHouseCluster(__file__) @@ -37,17 +30,8 @@ node2 = cluster.add_instance( macros={"shard": 1, "replica": 2}, ) -uuid_regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") - -def assert_create_query(nodes, table_name, expected): - replace_uuid = lambda x: re.sub(uuid_regex, "uuid", x) - query = "SHOW CREATE TABLE {}".format(table_name) - for node in nodes: - assert_eq_with_retry(node, query, expected, get_result=replace_uuid) - - -@pytest.fixture(scope="module", autouse=True) +@pytest.fixture(scope="session", autouse=True) def started_cluster(): try: cluster.start() @@ -57,11 +41,6 @@ def started_cluster(): cluster.shutdown() -@pytest.fixture(scope="module", autouse=True) -def setup_tables(started_cluster): - print(node.query("SELECT version()")) - - """ ### TESTS +1. Append mode @@ -86,24 +65,10 @@ def setup_tables(started_cluster): 19. Not existent BOTH tables (ON CLUSTER) +20. retry failed 21. overflow with wait test -22. ON CLUSTER (do when database replicated support is ready) +22. ON CLUSTER +SYSTEM STOP|START|REFRESH|CANCEL VIEW -SYSTEM WAIT VIEW [db.]name - -view_refreshes: -+progress -+elapsed -refresh_count -+exception - -+progress: inf if reading from table -+1 if reading from numbers() - -RMV settings -+ * `refresh_retries` - How many times to retry if refresh query fails with an exception. If all retries fail, skip to the next scheduled refresh time. 0 means no retries, -1 means infinite retries. Default: 0. - * `refresh_retry_initial_backoff_ms` - Delay before the first retry, if `refresh_retries` is not zero. Each subsequent retry doubles the delay, up to `refresh_retry_max_backoff_ms`. Default: 100 ms. - * `refresh_retry_max_backoff_ms` - Limit on the exponential growth of delay between refresh attempts. Default: 60000 ms (1 minute). ++SYSTEM WAIT VIEW [db.]name """ @@ -157,7 +122,7 @@ RMV_TEMPLATE = """{{ refresh_interval }} {% if select_query %} AS {{ select_query }}{% endif %} """ -CREATE_RMV_TEMPLATE = j2_template( +CREATE_RMV = j2_template( """CREATE MATERIALIZED VIEW {% if if_not_exists %}IF NOT EXISTS{% endif %} {% if db %}{{db}}.{% endif %}{{ table_name }} @@ -167,8 +132,7 @@ REFRESH + RMV_TEMPLATE ) -# ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] -ALTER_RMV_TEMPLATE = j2_template( +ALTER_RMV = j2_template( """ALTER TABLE {% if db %}{{db}}.{% endif %}{{ table_name }} {% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} @@ -178,84 +142,105 @@ MODIFY REFRESH ) -@pytest.fixture(scope="module") -def module_setup_tables(): - node.query(f"CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER test_cluster ENGINE = Atomic") +@pytest.fixture(scope="module", autouse=True) +def module_setup_tables(started_cluster): + node.query(f"DROP DATABASE IF EXISTS test_db") + node.query(f"CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS src2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER default") node.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" ) node.query( - f"CREATE TABLE src2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + f"CREATE TABLE src2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" ) node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" ) node.query( - f"CREATE TABLE tgt2 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" + f"CREATE TABLE tgt2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" ) node.query( - f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER test_cluster " + f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER default " f"REFRESH EVERY 10 HOUR engine Memory EMPTY AS select number as x from numbers(1)" ) - yield - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS src2 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER test_cluster") - node.query("DROP DATABASE IF EXISTS test_db ON CLUSTER test_cluster") @pytest.fixture(scope="function") def fn_setup_tables(): - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER test_cluster") - yield + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER default") + + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" + ) + + node.query( + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = Memory" + ) + node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") -def test_simple_append(module_setup_tables, fn_setup_tables): - create_sql = CREATE_RMV_TEMPLATE.render( +@pytest.mark.parametrize( + "select_query", + [ + "SELECT now() as a, number as b FROM numbers(2)", + "SELECT now() as a, b as b FROM src1", + ], +) +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +def test_simple_append( + module_setup_tables, + fn_setup_tables, + select_query, + with_append, + empty, +): + create_sql = CREATE_RMV.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", to_clause="tgt1", - select_query="SELECT now() as a, number as b FROM numbers(2)", - with_append=True, - empty=False, + select_query=select_query, + with_append=with_append, + empty=empty, ) node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv["exception"] is None + records = node.query("SELECT count() FROM test_rmv") + if empty: + assert records == "0\n" + else: + assert records == "2\n" + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(2) - rmv2 = get_rmv_info( - node, - "test_rmv", - ) + rmv2 = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv2["exception"] is None + if empty: + expect = "2\n" -def test_simple_append_from_table(fn2_setup_tables): - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - refresh_interval="EVERY 1 HOUR", - to_clause="tgt1", - select_query="SELECT now() as a, b as b FROM src1", - with_append=True, + if not with_append: + expect = "2\n" + + if with_append and not empty: + expect = "4\n" + + records = node.query_with_retry( + "SELECT count() FROM test_rmv", check_callback=lambda x: x == expect ) - node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(2) - rmv2 = get_rmv_info( - node, - "test_rmv", - ) - - assert rmv2["exception"] is None + assert records == expect @pytest.mark.parametrize("with_append", [True, False]) @@ -263,7 +248,7 @@ def test_simple_append_from_table(fn2_setup_tables): @pytest.mark.parametrize("on_cluster", [True, False]) @pytest.mark.parametrize("depends_on", [None, ["default.dummy_rmv"]]) @pytest.mark.parametrize("empty", [True, False]) -@pytest.mark.parametrize("database_name", [None, "default", "test_db"]) +@pytest.mark.parametrize("database_name", [None, "test_db"]) @pytest.mark.parametrize( "settings", [ @@ -278,7 +263,6 @@ def test_simple_append_from_table(fn2_setup_tables): def test_alters( module_setup_tables, fn_setup_tables, - started_cluster, with_append, if_not_exists, on_cluster, @@ -290,7 +274,7 @@ def test_alters( """ Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... """ - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", if_not_exists=if_not_exists, db=database_name, @@ -299,12 +283,12 @@ def test_alters( to_clause="tgt1", select_query="SELECT * FROM src1", with_append=with_append, - on_cluster="test_cluster" if on_cluster else None, + on_cluster="default" if on_cluster else None, settings=settings, ) node.query(create_sql) - # Check same RMV is created on cluster + # Check same RMV is created on whole cluster def compare_DDL_on_all_nodes(): show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") @@ -316,14 +300,15 @@ def test_alters( compare_DDL_on_all_nodes() maybe_db = f"{database_name}." if database_name else "" - node.query(f"DROP TABLE {maybe_db}test_rmv {'ON CLUSTER test_cluster' if on_cluster else ''}") + node.query( + f"DROP TABLE {maybe_db}test_rmv {'ON CLUSTER default' if on_cluster else ''}" + ) node.query(create_sql) compare_DDL_on_all_nodes() show_create = node.query(f"SHOW CREATE {maybe_db}test_rmv") - # Alter of RMV replaces all non-specified - alter_sql = ALTER_RMV_TEMPLATE.render( + alter_sql = ALTER_RMV.render( table_name="test_rmv", if_not_exists=if_not_exists, db=database_name, @@ -332,7 +317,7 @@ def test_alters( # can't change select with alter # select_query="SELECT * FROM src1", with_append=with_append, - on_cluster="test_cluster" if on_cluster else None, + on_cluster="default" if on_cluster else None, settings=settings, ) @@ -342,19 +327,134 @@ def test_alters( compare_DDL_on_all_nodes() +@pytest.mark.parametrize( + "append", + [True, False], +) +@pytest.mark.parametrize( + "empty", + [True, False], +) +@pytest.mark.parametrize( + "to_clause", + [ + (None, "tgt1", "tgt1"), + ("Engine MergeTree ORDER BY tuple()", None, "test_rmv"), + ], +) +def test_real_wait_refresh( + fn_setup_tables, + append, + empty, + to_clause, +): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers (too slow)") + + table_clause, to_clause_, tgt = to_clause + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 10 SECOND", + to_clause=to_clause_, + table_clause=table_clause, + select_query="SELECT now() as a, b FROM src1", + with_append=append, + empty=empty, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + + expected_rows = 0 + if empty: + expect_rows(expected_rows, table=tgt) + else: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], + # wait for refresh a little bit more than 10 seconds + max_attempts=12, + delay=1, + ) + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv2["exception"] is None + assert rmv2["status"] == "Scheduled" + assert rmv2["last_success_time"] == rmv["next_refresh_time"] + assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] + assert rmv2["retry"] == 0 + assert rmv2["read_rows"] == 2 + assert rmv2["read_bytes"] == 16 + assert rmv2["total_rows"] == 2 + assert rmv2["written_rows"] == 2 + assert rmv2["written_bytes"] == 24 + + node.query("SYSTEM STOP VIEW test_rmv") + time.sleep(12) + rmv3 = get_rmv_info(node, "test_rmv") + # no refresh happen + assert rmv3["status"] == "Disabled" + + del rmv3["status"] + del rmv2["status"] + assert rmv3 == rmv2 + + node.query("SYSTEM START VIEW test_rmv") + time.sleep(1) + rmv4 = get_rmv_info(node, "test_rmv") + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv4["exception"] is None + assert rmv4["status"] == "Scheduled" + assert rmv4["retry"] == 0 + assert rmv2["read_rows"] == 2 + assert rmv2["read_bytes"] == 16 + assert rmv2["total_rows"] == 2 + assert rmv2["written_rows"] == 2 + assert rmv2["written_bytes"] == 24 + + node.query("SYSTEM REFRESH VIEW test_rmv") + time.sleep(1) + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + def get_rmv_info( - node, table, condition=None, max_attempts=50, delay=0.3, wait_status="Scheduled" + node, + table, + condition=None, + max_attempts=50, + delay=0.3, + wait_status=None, ): def inner(): rmv_info = node.query_with_retry( f"SELECT * FROM system.view_refreshes WHERE view='{table}'", - check_callback=(lambda r: r.iloc[0]["status"] == wait_status) - if wait_status - else (lambda x: True), + check_callback=( + (lambda r: r.iloc[0]["status"] == wait_status) + if wait_status + else (lambda x: True) + ), parse=True, ).to_dict("records")[0] - # Is it time for python clickhouse-driver? rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) rmv_info["last_success_time"] = parse_ch_datetime(rmv_info["last_success_time"]) rmv_info["last_refresh_time"] = parse_ch_datetime(rmv_info["last_refresh_time"]) @@ -375,100 +475,6 @@ def parse_ch_datetime(date_str): return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") -# Add cases for: -# Randomize bigger than interval -# executed even with empty -"AFTER 1 MINUTE RANDOMIZE FOR 3 YEAR", -"EVERY 1 MINUTE OFFSET 1 SECOND RANDOMIZE FOR 3 YEAR", - -# special case -"EVERY 999 MINUTE" - -# should be restricted: -# Two different units in EVERY -"EVERY 1 YEAR 0 MONTH", -"EVERY 1 YEAR 1 MONTH", -"EVERY 1 YEAR 2 MONTH", -# "EVERY 1 YEAR 3 MONTH", -# "EVERY 1 YEAR 4 MONTH", -# "EVERY 1 YEAR 8 MONTH", -# "EVERY 1 YEAR 9 MONTH", -# "EVERY 1 YEAR 10 MONTH", -# "EVERY 1 YEAR 11 MONTH", -"EVERY 1 YEAR 12 MONTH", -# "EVERY 1 YEAR 13 MONTH", -# "EVERY 1 DAY", - -# should be restricted? -"EVERY 1 YEAR 2 YEAR", -"EVERY 1 MONTH 2 MONTH", -"AFTER 1 YEAR 2 YEAR", -"AFTER 1 MONTH 2 MONTH", - -"EVERY 1 DAY 0 HOUR", -"EVERY 1 DAY 1 HOUR", -"EVERY 1 DAY 2 HOUR", -"EVERY 1 DAY 23 HOUR", -"EVERY 1 DAY 24 HOUR", -"EVERY 1 DAY 28 HOUR", -# "EVERY 1 DAY 29 HOUR", -# "EVERY 1 DAY 30 HOUR", -# "EVERY 1 DAY 31 HOUR", -"EVERY 1 DAY 32 HOUR", - -# Interval shouldn't contain both calendar units and clock units (e.g. months and days) -# "EVERY 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", - - -INTERVALS_EVERY = [ - "EVERY 60 SECOND", - "EVERY 1 MINUTE", - "EVERY 1 HOUR", - "EVERY 1 DAY", - "EVERY 1 MONTH", - "EVERY 1 YEAR", - "EVERY 1 WEEK RANDOMIZE FOR 1 HOUR", - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", - "EVERY 1 MONTH OFFSET 1 DAY RANDOMIZE FOR 10 HOUR", - "EVERY 1 WEEK", - "EVERY 2 WEEK", - "EVERY 3 WEEK", - "EVERY 4 WEEK", - "EVERY 5 WEEK", -] - -INTERVALS_AFTER = [ - "AFTER 59 SECOND", - "AFTER 60 SECOND", - "AFTER 61 SECOND", - "AFTER 30 MINUTE", - "AFTER 9999 MINUTE", - "AFTER 2 HOUR", - "AFTER 2 DAY", - "AFTER 2 WEEK", - "AFTER 2 MONTH", - "AFTER 2 YEAR", - "AFTER 2 MONTH 3 DAY", - "AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", - "AFTER 1 YEAR RANDOMIZE FOR 11 MONTH", - "AFTER 1 MONTH", - "AFTER 1 MONTH 0 DAY", - "AFTER 1 MONTH 1 DAY", - "AFTER 1 MONTH 3 DAY", - "AFTER 1 MONTH 50 DAY", - "AFTER 1 YEAR 10 MONTH", - "AFTER 1 YEAR 10 MONTH 3 DAY", - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR", - "AFTER 1 YEAR 10 MONTH 3 DAY 7 HOUR 5 MINUTE 30 SECOND", -] - - -@pytest.fixture(scope="function") -def rmv_schedule_teardown(): - node.query("DROP TABLE IF EXISTS test_rmv") - - def expect_rows(rows, table="test_rmv"): inserted_data = node.query_with_retry( f"SELECT * FROM {table}", @@ -479,249 +485,15 @@ def expect_rows(rows, table="test_rmv"): assert len(inserted_data) == rows -@pytest.mark.parametrize("interval", INTERVALS_AFTER + INTERVALS_EVERY) -@pytest.mark.parametrize( - "append", - [True, False], -) -@pytest.mark.parametrize( - "empty", - [True, False], -) -def test_rmv_scheduling( - rmv_schedule_teardown, - interval, - append, - empty, -): - """ - Test creates RMV with multiple intervals, checks correctness of 'next_refresh_time' using schedule model, - check that data correctly written, set fake time and check all again. - """ - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - refresh_interval=interval, - table_clause="ENGINE = Memory", - select_query="SELECT now() as a, number as b FROM numbers(2)", - with_append=append, - empty=empty, - ) - node.query(create_sql) - - now = datetime.utcnow() - rmv = get_rmv_info(node, "test_rmv") - predicted_next_refresh_time = get_next_refresh_time(interval, now, first_week=True) - compare_dates( - rmv["next_refresh_time"], predicted_next_refresh_time, first_week=True - ) - assert rmv["next_refresh_time"] > now - assert rmv["exception"] is None - - if empty: - # No data is inserted with empty - expect_rows(0) - else: - # Query is executed one time - compare_dates_(rmv["last_success_time"], now) - expect_rows(2) - - # TODO: SET FAKE TIME doesn't work for these - if "RANDOMIZE" in interval: - return - if "OFFSET" in interval: - return - - # Trigger refresh, update `next_refresh_time` and check interval again - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - predicted_next_refresh_time2 = get_next_refresh_time( - interval, rmv["next_refresh_time"] - ) - rmv2 = get_rmv_info( - node, - "test_rmv", - condition=lambda x: x["next_refresh_time"] != rmv["next_refresh_time"], - ) - compare_dates(rmv2["next_refresh_time"], predicted_next_refresh_time2) - assert rmv["exception"] is None, rmv - - if append and not empty: - expect_rows(4) - else: - expect_rows(2) - - compare_dates( - rmv2["last_success_time"], predicted_next_refresh_time, first_week=True - ) - - -@pytest.fixture(scope="function") -def fn2_setup_tables(): - node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" - ) - - node.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") - - yield - node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - - -@pytest.mark.parametrize( - "append", - [True, False], -) -# @pytest.mark.parametrize("on_cluster", [True, False]) -@pytest.mark.parametrize( - "empty", - [True, False], -) -@pytest.mark.parametrize( - "to_clause", - [(None, "tgt1", "tgt1"), ("Engine MergeTree ORDER BY tuple()", None, "test_rmv")], -) -def test_real_wait_refresh( - fn2_setup_tables, - started_cluster, - append, - # on_cluster, - empty, - to_clause, -): - table_clause, to_clause_, tgt = to_clause - - create_sql = CREATE_RMV_TEMPLATE.render( - table_name="test_rmv", - # if_not_exists=if_not_exists, - refresh_interval="EVERY 8 SECOND", - to_clause=to_clause_, - table_clause=table_clause, - select_query="SELECT a as a, b FROM src1", - with_append=append, - # on_cluster="test_cluster" if on_cluster else None, - empty=empty - # settings={'setting1':'value1', 'setting2': 'value2'}, - ) - node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - - append_expected_rows = 0 - if not empty: - append_expected_rows += 2 - - if empty: - expect_rows(0, table=tgt) - else: - expect_rows(2, table=tgt) - - rmv2 = get_rmv_info( - node, - "test_rmv", - condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], - # wait for refresh a little bit more than 8 seconds - max_attempts=10, - delay=1, - ) - - if append: - append_expected_rows += 2 - expect_rows(append_expected_rows, table=tgt) - else: - expect_rows(2, table=tgt) - - assert rmv2["exception"] is None - assert rmv2["refresh_count"] == 2 - assert rmv2["status"] == "Scheduled" - assert rmv2["last_refresh_result"] == "Finished" - # assert rmv2["progress"] == , 1rmv2 - assert rmv2["last_success_time"] == rmv["next_refresh_time"] - assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] - assert rmv2["retry"] == 0 - assert rmv2["read_rows"] == 2 - assert rmv2["read_bytes"] == 24 - assert rmv2["total_rows"] == 0 - assert rmv2["total_bytes"] == 0 - assert rmv2["written_rows"] == 0 - assert rmv2["written_bytes"] == 0 - assert rmv2["result_rows"] == 0 - assert rmv2["result_bytes"] == 0 - - node.query("SYSTEM STOP VIEW test_rmv") - time.sleep(10) - rmv3 = get_rmv_info(node, "test_rmv") - # no refresh happen - assert rmv3["status"] == "Disabled" - - del rmv3["status"] - del rmv2["status"] - assert rmv3 == rmv2 - - node.query("SYSTEM START VIEW test_rmv") - time.sleep(1) - rmv4 = get_rmv_info(node, "test_rmv") - - if append: - append_expected_rows += 2 - expect_rows(append_expected_rows, table=tgt) - else: - expect_rows(2, table=tgt) - - assert rmv4["exception"] is None - assert rmv4["refresh_count"] == 3 - assert rmv4["status"] == "Scheduled" - assert rmv4["retry"] == 0 - assert rmv4["read_rows"] == 2 - assert rmv4["read_bytes"] == 24 - # why 0? - assert rmv4["total_rows"] == 0 - assert rmv4["total_bytes"] == 0 - assert rmv4["written_rows"] == 0 - assert rmv4["written_bytes"] == 0 - assert rmv4["result_rows"] == 0 - assert rmv4["result_bytes"] == 0 - - node.query("SYSTEM REFRESH VIEW test_rmv") - time.sleep(1) - if append: - append_expected_rows += 2 - expect_rows(append_expected_rows, table=tgt) - else: - expect_rows(2, table=tgt) - - -@pytest.fixture(scope="function") -def fn3_setup_tables(): - node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = MergeTree ORDER BY tuple()" - ) - node.query( - f"CREATE TABLE src1 ON CLUSTER test_cluster (a DateTime, b UInt64) ENGINE = Memory" - ) - node.query(f"INSERT INTO src1 SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)") - - yield - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS src1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") - - -@pytest.mark.parametrize( - "select_query", - ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], -) -def test_long_query(fn3_setup_tables, select_query): +def test_long_query(fn_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", to_clause="tgt1", - select_query=select_query, + select_query="SELECT now() a, sleep(1) b from numbers(10) settings max_block_size=1", with_append=False, empty=True, ) @@ -730,78 +502,76 @@ def test_long_query(fn3_setup_tables, select_query): node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") wait_seconds = 0 + progresses = [] while wait_seconds < 30: - time.sleep(1) - wait_seconds += 1 - rmv = get_rmv_info(node, "test_rmv", wait_status=None) - logging.info(rmv) + rmv = get_rmv_info(node, "test_rmv") if rmv["progress"] == 1: break + time.sleep(0.01) + wait_seconds += 0.01 + + # didn't start yet + if rmv["status"] == "Scheduled" or rmv["progress"] == 0: + continue + assert rmv["status"] == "Running" - assert 0 < rmv["progress"] < 1 - assert 0 < rmv["read_rows"] < 1000000000 - assert 0 < rmv["read_bytes"] < 8000000000 + assert 0 < rmv["read_rows"] < 100000000 + assert 0 < rmv["read_bytes"] < 1200000000 + progresses.append(rmv["progress"]) assert rmv["progress"] == 1 assert rmv["exception"] is None - assert 3 <= wait_seconds <= 30, wait_seconds - assert rmv["duration_ms"] >= 3000 - assert rmv["total_rows"] == 1000000000 - assert rmv["read_rows"] == 1000000000 - assert 0 < rmv["read_bytes"] == 8000000000 + + assert any(0 < p < 1 for p in progresses) -@pytest.mark.parametrize( - "select_query", - ["SELECT toDateTime(1) a, 1 b FROM numbers(1000000000)", "SELECT * FROM src1"], -) -def test_long_query_cancel(fn3_setup_tables, select_query): +def test_long_query_cancel(fn_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", - refresh_interval="EVERY 1 HOUR", + refresh_interval="EVERY 3 SECONDS", to_clause="tgt1", - select_query=select_query, + select_query="SELECT now() a, sleep(1) b from numbers(5) settings max_block_size=1", with_append=False, empty=True, + settings={"refresh_retries": "0"}, ) node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(1) + get_rmv_info(node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Running") node.query("SYSTEM CANCEL VIEW test_rmv") - time.sleep(1) - rmv = get_rmv_info(node, "test_rmv", wait_status=None) + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv["status"] == "Scheduled" - assert rmv["last_refresh_result"] == "Cancelled" - assert 0 < rmv["progress"] < 1 - assert 0 < rmv["read_rows"] < 1000000000 - assert 0 < rmv["read_bytes"] < 8000000000 - + assert rmv["exception"] == "cancelled" assert rmv["last_success_time"] is None - assert rmv["duration_ms"] > 0 - assert rmv["total_rows"] == 1000000000 + + assert node.query("SELECT count() FROM tgt1") == "0\n" + + get_rmv_info(node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Running") + get_rmv_info( + node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Scheduled" + ) + + assert node.query("SELECT count() FROM tgt1") == "5\n" @pytest.fixture(scope="function") -def fn4_setup_tables(): - node.query( - f"CREATE TABLE tgt1 ON CLUSTER test_cluster (a DateTime) ENGINE = Memory" - ) - yield - node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER test_cluster") - node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER test_cluster") +def fn3_setup_tables(): + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv ON CLUSTER default") + + node.query(f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime) ENGINE = Memory") -def test_query_fail(fn4_setup_tables): +def test_query_fail(fn3_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", refresh_interval="EVERY 1 HOUR", to_clause="tgt1", @@ -827,41 +597,30 @@ def test_query_fail(fn4_setup_tables): ) -def test_query_retry(fn4_setup_tables): +def test_query_retry(fn3_setup_tables): if node.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - create_sql = CREATE_RMV_TEMPLATE.render( + create_sql = CREATE_RMV.render( table_name="test_rmv", - refresh_interval="EVERY 1 HOUR", + refresh_interval="EVERY 2 SECOND", to_clause="tgt1", - select_query="SELECT throwIf(1, '111') a", # exception in 4/5 calls + select_query="SELECT throwIf(1, '111') a", with_append=False, empty=True, settings={ - "refresh_retries": "10", # TODO -1 - "refresh_retry_initial_backoff_ms": "10", - "refresh_retry_max_backoff_ms": "10", + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "1", + "refresh_retry_max_backoff_ms": "1", }, ) node.query(create_sql) - rmv = get_rmv_info(node, "test_rmv") - node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") - time.sleep(20) - - rmv2 = get_rmv_info( + rmv = get_rmv_info( node, "test_rmv", - wait_status=None, + delay=0.1, + max_attempts=1000, + condition=lambda x: x["retry"] == 11, ) - assert rmv2["last_refresh_result"] == "Error" - assert rmv2["retry"] == 10 - assert False - - -# def test_query_retry_success(fn4_setup_tables): -# TODO: SELECT throwIf(rand() % 10 != 0, '111') a -# pass - - -# TODO -1 + assert rmv["retry"] == 11 + assert "FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" in rmv["exception"] diff --git a/tests/integration/test_refreshable_mat_view/test_schedule_model.py b/tests/integration/test_refreshable_mat_view/test_schedule_model.py deleted file mode 100644 index da7978918e4..00000000000 --- a/tests/integration/test_refreshable_mat_view/test_schedule_model.py +++ /dev/null @@ -1,89 +0,0 @@ -from datetime import datetime - -from test_refreshable_mat_view.schedule_model import get_next_refresh_time - - -def get_next_refresh_time_(*args, **kwargs): - return get_next_refresh_time(*args, **kwargs)["time"] - - -def test_refresh_schedules(): - t = datetime(2000, 1, 1, 1, 1, 1) - - assert get_next_refresh_time_("EVERY 1 SECOND", t) == datetime(2000, 1, 1, 1, 1, 2) - assert get_next_refresh_time_("EVERY 1 MINUTE", t) == datetime( - 2000, - 1, - 1, - 1, - 2, - ) - assert get_next_refresh_time_("EVERY 1 HOUR", t) == datetime( - 2000, - 1, - 1, - 2, - ) - assert get_next_refresh_time_("EVERY 1 DAY", t) == datetime(2000, 1, 2) - assert get_next_refresh_time_("EVERY 1 WEEK", t) == datetime(2000, 1, 10) - assert get_next_refresh_time_("EVERY 2 WEEK", t) == datetime(2000, 1, 17) - assert get_next_refresh_time_("EVERY 1 MONTH", t) == datetime(2000, 2, 1) - assert get_next_refresh_time_("EVERY 1 YEAR", t) == datetime(2001, 1, 1) - - assert get_next_refresh_time_("EVERY 3 YEAR 4 MONTH 10 DAY", t) == datetime( - 2003, 5, 11 - ) - - # OFFSET - assert get_next_refresh_time_( - "EVERY 1 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - ) == datetime(2000, 2, 6, 2, 30, 15) - assert get_next_refresh_time_( - "EVERY 1 YEAR 2 MONTH OFFSET 5 DAY 2 HOUR 30 MINUTE 15 SECOND", t - ) == datetime(2001, 3, 6, 2, 30, 15) - - assert get_next_refresh_time_( - "EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE", t - ) == datetime(2000, 1, 22, 15, 10) - - # AFTER - assert get_next_refresh_time_("AFTER 30 SECOND", t) == datetime( - 2000, 1, 1, 1, 1, 31 - ) - assert get_next_refresh_time_("AFTER 30 MINUTE", t) == datetime( - 2000, 1, 1, 1, 31, 1 - ) - assert get_next_refresh_time_("AFTER 2 HOUR", t) == datetime(2000, 1, 1, 3, 1, 1) - assert get_next_refresh_time_("AFTER 2 DAY", t) == datetime(2000, 1, 3, 1, 1, 1) - assert get_next_refresh_time_("AFTER 2 WEEK", t) == datetime(2000, 1, 15, 1, 1, 1) - assert get_next_refresh_time_("AFTER 2 MONTH", t) == datetime(2000, 3, 1, 1, 1, 1) - assert get_next_refresh_time_("AFTER 2 YEAR", t) == datetime(2002, 1, 1, 1, 1, 1) - - assert get_next_refresh_time_("AFTER 2 YEAR 1 MONTH", t) == datetime( - 2002, 2, 1, 1, 1, 1 - ) - - assert get_next_refresh_time_("AFTER 1 MONTH 2 YEAR", t) == datetime( - 2002, 2, 1, 1, 1, 1 - ) - - # RANDOMIZE - next_refresh = get_next_refresh_time_( - "EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR", t - ) - - assert next_refresh == (datetime(2000, 1, 2, 1, 30), datetime(2000, 1, 2, 2, 30)) - - next_refresh = get_next_refresh_time_( - "EVERY 2 MONTH 3 DAY 5 HOUR OFFSET 3 HOUR 20 SECOND RANDOMIZE FOR 3 DAY 1 HOUR", - t, - ) - assert next_refresh == ( - datetime(2000, 3, 2, 19, 30, 20), - datetime(2000, 3, 5, 20, 30, 20), - ) - - assert get_next_refresh_time_("AFTER 2 MONTH 3 DAY RANDOMIZE FOR 1 DAY", t) == ( - datetime(2000, 3, 3, 13, 1, 1), - datetime(2000, 3, 4, 13, 1, 1), - ) diff --git a/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml b/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml deleted file mode 100644 index 7c0e60a044e..00000000000 --- a/tests/integration/test_refreshable_mat_view_db_replicated/configs/settings.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - 1 - 1 - 1 - 0 - 0 - - - - - default - - - diff --git a/tests/integration/test_refreshable_mat_view_replicated/__init__.py b/tests/integration/test_refreshable_mat_view_replicated/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_refreshable_mat_view_replicated/configs/config.xml b/tests/integration/test_refreshable_mat_view_replicated/configs/config.xml new file mode 100644 index 00000000000..fdaad39175f --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/configs/config.xml @@ -0,0 +1,3 @@ + + Etc/UTC + diff --git a/tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml b/tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml new file mode 100644 index 00000000000..df6377287c3 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/configs/remote_servers.xml @@ -0,0 +1,16 @@ + + + + + + node1_1 + 9000 + + + node1_2 + 9000 + + + + + diff --git a/tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml b/tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml new file mode 100644 index 00000000000..d15acfaa303 --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/configs/settings.xml @@ -0,0 +1,8 @@ + + + + 1 + Etc/UTC + + + diff --git a/tests/integration/test_refreshable_mat_view_replicated/test.py b/tests/integration/test_refreshable_mat_view_replicated/test.py new file mode 100644 index 00000000000..4293a3ca23b --- /dev/null +++ b/tests/integration/test_refreshable_mat_view_replicated/test.py @@ -0,0 +1,625 @@ +import datetime +import logging +import time +from datetime import datetime +from typing import Optional + +import pytest +from jinja2 import Environment, Template + +import helpers.client +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import wait_condition + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node1_1", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + keeper_required_feature_flags=["multi_read", "create_if_not_exists"], + macros={"shard": 1, "replica": 1}, +) +node2 = cluster.add_instance( + "node1_2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/settings.xml"], + with_zookeeper=True, + stay_alive=True, + keeper_required_feature_flags=["multi_read", "create_if_not_exists"], + macros={"shard": 1, "replica": 2}, +) + +nodes = [node, node2] + + +@pytest.fixture(scope="session", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +""" +### TESTS ++1. Append mode +2. Restart node and wait for restore ++3. Simple functional testing: all values in refresh result correct (two and more rmv) ++4. Combinations of intervals ++5. Two (and more) rmv from single to single [APPEND] ++6. ALTER rmv ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...] [SETTINGS ...] ++7. RMV without tgt table (automatic table) (check APPEND) + ++8 DROP rmv ++9 CREATE - DROP - CREATE - ALTER +11. Long queries over refresh time (check settings) +13. incorrect intervals (interval 1 sec, offset 1 minute) + - OFFSET less than the period. 'EVERY 1 MONTH OFFSET 5 WEEK' + - cases below ++14. ALTER on cluster + +15. write to distributed with / without APPEND +17. Not existent TO table (ON CLUSTER) +18. Not existent FROM table (ON CLUSTER) +19. Not existent BOTH tables (ON CLUSTER) ++20. retry failed +21. overflow with wait test +22. ON CLUSTER + ++SYSTEM STOP|START|REFRESH|CANCEL VIEW ++SYSTEM WAIT VIEW [db.]name + +""" + + +def j2_template( + string: str, + globals: Optional[dict] = None, + filters: Optional[dict] = None, + tests: Optional[dict] = None, +) -> Template: + def uppercase(value: str): + return value.upper() + + def lowercase(value: str): + return value.lower() + + def format_settings(items: dict): + return ", ".join([f"{k}={v}" for k, v in items.items()]) + + # Create a custom environment and add the functions + env = Environment( + trim_blocks=False, lstrip_blocks=True, keep_trailing_newline=False + ) + env.globals["uppercase"] = uppercase + env.globals["lowercase"] = lowercase + env.filters["format_settings"] = format_settings + + if filters: + env.filters.update(filters) + if globals: + env.globals.update(globals) + if tests: + env.tests.update(tests) + + return env.from_string(string) + + +def assert_same_values(lst: list): + if not isinstance(lst, list): + lst = list(lst) + assert all(x == lst[0] for x in lst) + + +RMV_TEMPLATE = """{{ refresh_interval }} +{% if depends_on %}DEPENDS ON {{ depends_on|join(', ') }}{% endif %} +{% if settings %}SETTINGS {{ settings|format_settings }}{% endif %} +{% if with_append %}APPEND{% endif %} +{% if to_clause %}TO {{ to_clause }}{% endif %} +{% if table_clause %}{{ table_clause }}{% endif %} +{% if empty %}EMPTY{% endif %} +{% if select_query %} AS {{ select_query }}{% endif %} +""" + +CREATE_RMV = j2_template( + """CREATE MATERIALIZED VIEW +{% if if_not_exists %}IF NOT EXISTS{% endif %} +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +REFRESH +""" + + RMV_TEMPLATE +) + +ALTER_RMV = j2_template( + """ALTER TABLE +{% if db %}{{db}}.{% endif %}{{ table_name }} +{% if on_cluster %}ON CLUSTER {{ on_cluster }}{% endif %} +MODIFY REFRESH +""" + + RMV_TEMPLATE +) + + +@pytest.fixture(scope="module", autouse=True) +def module_setup_tables(started_cluster): + + # default is Atomic by default + node.query(f"DROP DATABASE IF EXISTS default ON CLUSTER default SYNC") + node.query( + "CREATE DATABASE IF NOT EXISTS default ON CLUSTER default ENGINE=Replicated('/clickhouse/default/','{shard}','{replica}')" + ) + + assert ( + node.query( + f"SELECT engine FROM clusterAllReplicas(default, system.databases) where name='default'" + ) + == "Replicated\nReplicated\n" + ) + + node.query(f"DROP DATABASE IF EXISTS test_db ON CLUSTER default SYNC") + node.query( + "CREATE DATABASE test_db ON CLUSTER default ENGINE=Replicated('/clickhouse/test_db/','{shard}','{replica}')" + ) + + assert ( + node.query( + f"SELECT engine FROM clusterAllReplicas(default, system.databases) where name='test_db'" + ) + == "Replicated\nReplicated\n" + ) + + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS src2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt2 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv") + + node.query( + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE src2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE tgt2 ON CLUSTER default (a DateTime, b UInt64) ENGINE = ReplicatedMergeTree() ORDER BY tuple()" + ) + node.query( + f"CREATE MATERIALIZED VIEW IF NOT EXISTS dummy_rmv ON CLUSTER default " + f"REFRESH EVERY 10 HOUR ENGINE = ReplicatedMergeTree() ORDER BY tuple() EMPTY AS select number as x from numbers(1)" + ) + + +@pytest.fixture(scope="function") +def fn_setup_tables(): + node.query("DROP TABLE IF EXISTS src1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv") + + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime, b UInt64) " + f"ENGINE = ReplicatedMergeTree ORDER BY tuple()" + ) + + node.query( + f"CREATE TABLE src1 ON CLUSTER default (a DateTime, b UInt64) " + f"ENGINE = ReplicatedMergeTree ORDER BY tuple()" + ) + node.query(f"INSERT INTO src1 VALUES ('2020-01-01', 1), ('2020-01-02', 2)") + + +@pytest.mark.parametrize( + "select_query", + [ + "SELECT now() as a, number as b FROM numbers(2)", + "SELECT now() as a, b as b FROM src1", + ], +) +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("empty", [True, False]) +def test_simple_append( + module_setup_tables, + fn_setup_tables, + select_query, + with_append, + empty, +): + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + select_query=select_query, + with_append=with_append, + on_cluster="default", + empty=empty, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") + assert rmv["exception"] is None + + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + + records = node.query("SELECT count() FROM test_rmv") + + if empty: + assert records == "0\n" + else: + assert records == "2\n" + + for n in nodes: + n.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + + rmv2 = get_rmv_info(node, "test_rmv", wait_status="Scheduled") + + assert rmv2["exception"] is None + + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + if empty: + expect = "2\n" + + if not with_append: + expect = "2\n" + + if with_append and not empty: + expect = "4\n" + + records = node.query_with_retry( + "SELECT count() FROM test_rmv", check_callback=lambda x: x == expect + ) + assert records == expect + + +@pytest.mark.parametrize("with_append", [True, False]) +@pytest.mark.parametrize("if_not_exists", [True, False]) +@pytest.mark.parametrize("depends_on", [None, ["default.dummy_rmv"]]) +@pytest.mark.parametrize("empty", [True, False]) +@pytest.mark.parametrize("database_name", ["test_db"]) # None, +@pytest.mark.parametrize( + "settings", + [ + {}, + { + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "10", + "refresh_retry_max_backoff_ms": "20", + }, + ], +) +def test_alters( + module_setup_tables, + fn_setup_tables, + with_append, + if_not_exists, + depends_on, + empty, + database_name, + settings, +): + """ + Check correctness of functional states of RMV after CREATE, DROP, ALTER, trigger of RMV, ... + """ + create_sql = CREATE_RMV.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db="test_db", + refresh_interval="EVERY 1 HOUR", + depends_on=depends_on, + to_clause="tgt1", + select_query="SELECT * FROM src1", + with_append=with_append, + settings=settings, + ) + node.query(create_sql) + + # Check same RMV is created on whole cluster + def compare_DDL_on_all_nodes(): + show_create_all_nodes = cluster.query_all_nodes("SHOW CREATE test_rmv") + assert_same_values(show_create_all_nodes.values()) + + compare_DDL_on_all_nodes() + + node.query(f"DROP TABLE test_db.test_rmv") + node.query(create_sql) + compare_DDL_on_all_nodes() + + show_create = node.query(f"SHOW CREATE test_db.test_rmv") + + alter_sql = ALTER_RMV.render( + table_name="test_rmv", + if_not_exists=if_not_exists, + db="test_db", + refresh_interval="EVERY 1 HOUR", + depends_on=depends_on, + # can't change select with alter + # select_query="SELECT * FROM src1", + with_append=with_append, + settings=settings, + ) + + node.query(alter_sql) + show_create_after_alter = node.query(f"SHOW CREATE test_db.test_rmv") + assert show_create == show_create_after_alter + compare_DDL_on_all_nodes() + + +@pytest.mark.parametrize( + "append", + [True, False], +) +@pytest.mark.parametrize( + "empty", + [True, False], +) +@pytest.mark.parametrize( + "to_clause", + [ + (None, "tgt1", "tgt1"), + ("Engine ReplicatedMergeTree ORDER BY tuple()", None, "test_rmv"), + ], +) +def test_real_wait_refresh( + fn_setup_tables, + append, + empty, + to_clause, +): + table_clause, to_clause_, tgt = to_clause + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 10 SECOND", + to_clause=to_clause_, + table_clause=table_clause, + select_query="SELECT now() as a, b FROM src1", + with_append=append, + on_cluster="default", + empty=empty, + ) + node.query(create_sql) + rmv = get_rmv_info(node, "test_rmv") + time.sleep(1) + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + + expected_rows = 0 + if empty: + expect_rows(expected_rows, table=tgt) + else: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + + rmv2 = get_rmv_info( + node, + "test_rmv", + condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], + # wait for refresh a little bit more than 10 seconds + max_attempts=12, + delay=1, + wait_status="Scheduled", + ) + + node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") + + rmv22 = get_rmv_info( + node, + "test_rmv", + wait_status="Scheduled", + ) + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv2["exception"] is None + assert rmv2["status"] == "Scheduled" + assert rmv2["last_success_time"] == rmv["next_refresh_time"] + assert rmv2["last_refresh_time"] == rmv["next_refresh_time"] + assert rmv2["retry"] == 0 and rmv22["retry"] == 0 + + for n in nodes: + n.query("SYSTEM STOP VIEW test_rmv") + time.sleep(12) + rmv3 = get_rmv_info(node, "test_rmv") + # no refresh happen + assert rmv3["status"] == "Disabled" + + del rmv3["status"] + del rmv2["status"] + assert rmv3 == rmv2 + + for n in nodes: + n.query("SYSTEM START VIEW test_rmv") + time.sleep(1) + rmv4 = get_rmv_info(node, "test_rmv") + + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + assert rmv4["exception"] is None + assert rmv4["status"] == "Scheduled" + assert rmv4["retry"] == 0 + + node.query("SYSTEM REFRESH VIEW test_rmv") + time.sleep(1) + if append: + expected_rows += 2 + expect_rows(expected_rows, table=tgt) + else: + expect_rows(2, table=tgt) + + +def get_rmv_info( + node, + table, + condition=None, + max_attempts=50, + delay=0.3, + wait_status=None, +): + def inner(): + rmv_info = node.query_with_retry( + f"SELECT * FROM system.view_refreshes WHERE view='{table}'", + check_callback=( + (lambda r: r.iloc[0]["status"] == wait_status) + if wait_status + else (lambda x: True) + ), + parse=True, + ).to_dict("records")[0] + + rmv_info["next_refresh_time"] = parse_ch_datetime(rmv_info["next_refresh_time"]) + rmv_info["last_success_time"] = parse_ch_datetime(rmv_info["last_success_time"]) + rmv_info["last_refresh_time"] = parse_ch_datetime(rmv_info["last_refresh_time"]) + logging.info(rmv_info) + return rmv_info + + if condition: + res = wait_condition(inner, condition, max_attempts=max_attempts, delay=delay) + return res + + res = inner() + return res + + +def parse_ch_datetime(date_str): + if date_str is None: + return None + return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") + + +def expect_rows(rows, table="test_rmv"): + inserted_data = node.query_with_retry( + f"SELECT * FROM {table}", + parse=True, + check_callback=lambda x: len(x) == rows, + retry_count=100, + ) + assert len(inserted_data) == rows + + +def test_long_query_cancel(fn_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 5 SECONDS", + to_clause="tgt1", + select_query="SELECT now() a, sleep(1) b from numbers(5) settings max_block_size=1", + with_append=False, + empty=True, + settings={"refresh_retries": "0"}, + ) + node.query(create_sql) + + done = False + start = time.time() + while not done: + for n in nodes: + n.query("SYSTEM CANCEL VIEW test_rmv") + if get_rmv_info(node2, "test_rmv")["exception"] == "cancelled": + done = True + + time.sleep(0.1) + if time.time() - start > 10: + raise AssertionError("Can't cancel query") + + rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") + assert rmv["status"] == "Scheduled" + assert rmv["exception"] == "cancelled" + assert rmv["last_success_time"] is None + + assert node.query("SELECT count() FROM tgt1") == "0\n" + + get_rmv_info(node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Running") + get_rmv_info( + node, "test_rmv", delay=0.1, max_attempts=1000, wait_status="Scheduled" + ) + + assert node.query("SELECT count() FROM tgt1") == "5\n" + + +@pytest.fixture(scope="function") +def fn3_setup_tables(): + node.query("DROP TABLE IF EXISTS tgt1 ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_rmv ON CLUSTER default") + node.query("DROP TABLE IF EXISTS test_db.test_rmv") + + node.query( + f"CREATE TABLE tgt1 ON CLUSTER default (a DateTime) ENGINE = ReplicatedMergeTree ORDER BY tuple()" + ) + + +def test_query_fail(fn3_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 1 HOUR", + to_clause="tgt1", + # Argument at index 1 for function throwIf must be constant + select_query="SELECT throwIf(1, toString(rand())) a", + with_append=False, + on_cluster="default", + empty=True, + settings={ + "refresh_retries": "10", + }, + ) + with pytest.raises(helpers.client.QueryRuntimeException) as exc: + node.query(create_sql) + assert "Argument at index 1 for function throwIf must be constant" in str( + exc.value + ) + assert ( + node.query(f"SELECT count() FROM system.view_refreshes WHERE view='test_rmv'") + == "0\n" + ) + assert ( + node.query(f"SELECT count() FROM system.tables WHERE name='test_rmv'") == "0\n" + ) + + +def test_query_retry(fn3_setup_tables): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + + create_sql = CREATE_RMV.render( + table_name="test_rmv", + refresh_interval="EVERY 2 SECOND", + to_clause="tgt1", + select_query="SELECT throwIf(1, '111') a", + with_append=False, + on_cluster="default", + empty=True, + settings={ + "refresh_retries": "10", + "refresh_retry_initial_backoff_ms": "1", + "refresh_retry_max_backoff_ms": "1", + }, + ) + node.query(create_sql) + rmv = get_rmv_info( + node, + "test_rmv", + delay=0.1, + max_attempts=1000, + condition=lambda x: x["retry"] == 11, + ) + assert rmv["retry"] == 11 + assert "FUNCTION_THROW_IF_VALUE_IS_NON_ZERO" in rmv["exception"] From eb42bbbf7493edb565fab3a92c7a621b3abac727 Mon Sep 17 00:00:00 2001 From: Nikita Fomichev Date: Wed, 4 Dec 2024 20:16:36 +0100 Subject: [PATCH 30/38] RMV: fix tests with replicated db --- tests/integration/helpers/test_tools.py | 5 ++- .../test_refreshable_mat_view/test.py | 4 +-- .../test.py | 34 +++++++++---------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index c334a7366da..bd43adad95d 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -186,6 +186,7 @@ def csv_compare(result, expected): def wait_condition(func, condition, max_attempts=10, delay=0.1): attempts = 0 + result = None while attempts < max_attempts: result = func() if condition(result): @@ -194,4 +195,6 @@ def wait_condition(func, condition, max_attempts=10, delay=0.1): if attempts < max_attempts: time.sleep(delay) - raise Exception(f"Function did not satisfy condition after {max_attempts} attempts") + raise Exception( + f"Function did not satisfy condition after {max_attempts} attempts. Last result:\n{result}" + ) diff --git a/tests/integration/test_refreshable_mat_view/test.py b/tests/integration/test_refreshable_mat_view/test.py index c25d37fdc19..ed8e9de1168 100644 --- a/tests/integration/test_refreshable_mat_view/test.py +++ b/tests/integration/test_refreshable_mat_view/test.py @@ -377,8 +377,8 @@ def test_real_wait_refresh( "test_rmv", condition=lambda x: x["last_refresh_time"] == rmv["next_refresh_time"], # wait for refresh a little bit more than 10 seconds - max_attempts=12, - delay=1, + max_attempts=30, + delay=0.5, ) if append: diff --git a/tests/integration/test_refreshable_mat_view_replicated/test.py b/tests/integration/test_refreshable_mat_view_replicated/test.py index 4293a3ca23b..cf09dc9c8e1 100644 --- a/tests/integration/test_refreshable_mat_view_replicated/test.py +++ b/tests/integration/test_refreshable_mat_view_replicated/test.py @@ -221,13 +221,19 @@ def fn_setup_tables(): @pytest.mark.parametrize( "select_query", [ - "SELECT now() as a, number as b FROM numbers(2)", - "SELECT now() as a, b as b FROM src1", + "SELECT now() as a, number as b FROM numbers(2) SETTINGS insert_deduplicate=0", + "SELECT now() as a, b as b FROM src1 SETTINGS insert_deduplicate=0", ], ) -@pytest.mark.parametrize("with_append", [True, False]) -@pytest.mark.parametrize("empty", [True, False]) -def test_simple_append( +@pytest.mark.parametrize( + "with_append", + [True, False], +) +@pytest.mark.parametrize( + "empty", + [True, False], +) +def test_append( module_setup_tables, fn_setup_tables, select_query, @@ -247,8 +253,6 @@ def test_simple_append( rmv = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv["exception"] is None - node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") - records = node.query("SELECT count() FROM test_rmv") if empty: @@ -256,20 +260,13 @@ def test_simple_append( else: assert records == "2\n" - for n in nodes: - n.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") + node.query(f"SYSTEM TEST VIEW test_rmv SET FAKE TIME '{rmv['next_refresh_time']}'") rmv2 = get_rmv_info(node, "test_rmv", wait_status="Scheduled") assert rmv2["exception"] is None - node.query("SYSTEM SYNC DATABASE REPLICA ON CLUSTER default default") - if empty: - expect = "2\n" - - if not with_append: - expect = "2\n" - + expect = "2\n" if with_append and not empty: expect = "4\n" @@ -373,6 +370,9 @@ def test_real_wait_refresh( empty, to_clause, ): + if node.is_built_with_sanitizer(): + pytest.skip("Disabled for sanitizers") + table_clause, to_clause_, tgt = to_clause create_sql = CREATE_RMV.render( @@ -380,7 +380,7 @@ def test_real_wait_refresh( refresh_interval="EVERY 10 SECOND", to_clause=to_clause_, table_clause=table_clause, - select_query="SELECT now() as a, b FROM src1", + select_query="SELECT now() as a, b FROM src1 SETTINGS insert_deduplicate=0", with_append=append, on_cluster="default", empty=empty, From 05d8584966b05ed2196fc1ffb3bf0ecbdf9ac910 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Dec 2024 10:34:36 +0000 Subject: [PATCH 31/38] Fix chassert --- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 980d2f479cb..5352fc641fa 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -128,6 +128,14 @@ bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & sec { return !(first == second); } + +bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Object & second) +{ + static String fields_key = "fields"; + if (!first.has(fields_key) || !second.has(fields_key)) + return false; + return first.getObject(fields_key) == second.getObject(fields_key); +} } @@ -481,7 +489,7 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem if (iceberg_table_schemas_by_ids.contains(schema_id)) { chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); - chassert(*iceberg_table_schemas_by_ids.at(schema_id) == *schema_ptr); + chassert(schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr)); } else { From 57d446ebfdb7c0aaed5c30292dcb2467a965d8d0 Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Dec 2024 10:44:22 +0000 Subject: [PATCH 32/38] Fix bug --- src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 5352fc641fa..8b83f6434db 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -134,7 +134,7 @@ bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Obj static String fields_key = "fields"; if (!first.has(fields_key) || !second.has(fields_key)) return false; - return first.getObject(fields_key) == second.getObject(fields_key); + return (*first.getObject(fields_key)) == (*second.getObject(fields_key)); } } From 72cc0645472136effec57e22e26344cefc0e0c0d Mon Sep 17 00:00:00 2001 From: divanik Date: Tue, 10 Dec 2024 11:07:48 +0000 Subject: [PATCH 33/38] Fix bug --- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 8b83f6434db..9cf4f64cdb3 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -107,7 +107,8 @@ std::pair parseDecimal(const String & type_name) return {precision, scale}; } -bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & second) +template +bool equals(const T & first, const T & second) { std::stringstream first_string_stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM std::stringstream second_string_stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM @@ -124,17 +125,23 @@ bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & sec return first_string_stream.str() == second_string_stream.str(); } -bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & second) + +bool operator==(const Poco::JSON::Object & first, const Poco::JSON::Object & second) { - return !(first == second); + return equals(first, second); +} + +bool operator==(const Poco::JSON::Array & first, const Poco::JSON::Array & second) +{ + return equals(first, second); } bool schemasAreIdentical(const Poco::JSON::Object & first, const Poco::JSON::Object & second) { static String fields_key = "fields"; - if (!first.has(fields_key) || !second.has(fields_key)) + if (!first.isArray(fields_key) || !second.isArray(fields_key)) return false; - return (*first.getObject(fields_key)) == (*second.getObject(fields_key)); + return *(first.getArray(fields_key)) == *(second.getArray(fields_key)); } } From fb02bec3dc07c15450d265d4ae93b964ab62f7c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 10 Dec 2024 12:22:51 +0100 Subject: [PATCH 34/38] Revert "Enable JIT compilation for more expressions" --- src/Core/AccurateComparison.h | 6 - src/DataTypes/IDataType.h | 4 - src/DataTypes/NumberTraits.h | 33 ---- src/Functions/DivisionUtils.h | 99 +--------- src/Functions/FunctionBinaryArithmetic.h | 121 ++++-------- src/Functions/FunctionUnaryArithmetic.h | 16 +- src/Functions/FunctionsComparison.h | 198 ++------------------ src/Functions/FunctionsConversion.h | 141 -------------- src/Functions/FunctionsLogical.cpp | 39 ---- src/Functions/FunctionsLogical.h | 101 +++------- src/Functions/abs.cpp | 57 +----- src/Functions/assumeNotNull.cpp | 21 --- src/Functions/bitCount.cpp | 25 +-- src/Functions/identity.h | 26 +-- src/Functions/isNotNull.cpp | 22 --- src/Functions/isNull.cpp | 20 -- src/Functions/sign.cpp | 45 +---- src/Interpreters/ActionsDAG.cpp | 12 +- src/Interpreters/ActionsDAG.h | 4 +- src/Interpreters/ExpressionActions.cpp | 9 - src/Interpreters/JIT/CHJIT.cpp | 4 - src/Interpreters/JIT/CompileDAG.cpp | 8 +- src/Processors/QueryPlan/ExpressionStep.cpp | 39 +--- 23 files changed, 112 insertions(+), 938 deletions(-) diff --git a/src/Core/AccurateComparison.h b/src/Core/AccurateComparison.h index 25937a61168..87ff14e40e7 100644 --- a/src/Core/AccurateComparison.h +++ b/src/Core/AccurateComparison.h @@ -209,14 +209,12 @@ template struct EqualsOp using SymmetricOp = EqualsOp; static UInt8 apply(A a, B b) { return accurate::equalsOp(a, b); } - static constexpr bool compilable = true; }; template struct NotEqualsOp { using SymmetricOp = NotEqualsOp; static UInt8 apply(A a, B b) { return accurate::notEqualsOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOp; @@ -225,14 +223,12 @@ template struct LessOp { using SymmetricOp = GreaterOp; static UInt8 apply(A a, B b) { return accurate::lessOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOp { using SymmetricOp = LessOp; static UInt8 apply(A a, B b) { return accurate::greaterOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOrEqualsOp; @@ -241,14 +237,12 @@ template struct LessOrEqualsOp { using SymmetricOp = GreaterOrEqualsOp; static UInt8 apply(A a, B b) { return accurate::lessOrEqualsOp(a, b); } - static constexpr bool compilable = true; }; template struct GreaterOrEqualsOp { using SymmetricOp = LessOrEqualsOp; static UInt8 apply(A a, B b) { return accurate::greaterOrEqualsOp(a, b); } - static constexpr bool compilable = true; }; } diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 2771d056e96..8f06526ddbb 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -555,7 +555,6 @@ inline bool isNullableOrLowCardinalityNullable(const DataTypePtr & data_type) template constexpr bool IsDataTypeDecimal = false; template constexpr bool IsDataTypeNumber = false; -template constexpr bool IsDataTypeNativeNumber = false; template constexpr bool IsDataTypeDateOrDateTime = false; template constexpr bool IsDataTypeDate = false; template constexpr bool IsDataTypeEnum = false; @@ -582,9 +581,6 @@ template constexpr bool IsDataTypeDecimal> = t template <> inline constexpr bool IsDataTypeDecimal = true; template constexpr bool IsDataTypeNumber> = true; -template -requires std::is_arithmetic_v -constexpr bool IsDataTypeNativeNumber> = true; template <> inline constexpr bool IsDataTypeDate = true; template <> inline constexpr bool IsDataTypeDate = true; diff --git a/src/DataTypes/NumberTraits.h b/src/DataTypes/NumberTraits.h index b4a322e1083..5f5962da5ee 100644 --- a/src/DataTypes/NumberTraits.h +++ b/src/DataTypes/NumberTraits.h @@ -205,39 +205,6 @@ struct ResultOfIf ConstructedType, Error>>>; }; -/** Type casting for `modulo` function: - * UInt, UInt -> UInt - * Int, Int -> Int - * UInt, Int -> Int - * UInt64, Int -> Error - * Float, Float -> Float64 - * Float, [U]Int -> Float64 - */ -template -struct ResultOfModuloNativePromotion -{ - static_assert(is_arithmetic_v && is_arithmetic_v); - - static constexpr bool has_float = std::is_floating_point_v || std::is_floating_point_v; - static constexpr bool has_integer = is_integer || is_integer; - static constexpr bool has_signed = is_signed_v || is_signed_v; - static constexpr bool has_unsigned = !is_signed_v || !is_signed_v; - - static constexpr size_t max_size_of_unsigned_integer = max(is_signed_v ? 0 : sizeof(A), is_signed_v ? 0 : sizeof(B)); - static constexpr size_t max_size_of_signed_integer = max(is_signed_v ? sizeof(A) : 0, is_signed_v ? sizeof(B) : 0); - static constexpr size_t max_size_of_integer = max(is_integer ? sizeof(A) : 0, is_integer ? sizeof(B) : 0); - - using ConstructedType = typename Construct< - has_signed, - false, - (has_signed ^ has_unsigned) ? max(max_size_of_unsigned_integer * 2, max_size_of_signed_integer) : max(sizeof(A), sizeof(B))>::Type; - - using Type = std::conditional_t< - std::is_same_v, - A, - std::conditional_t>>; -}; - /** Before applying operator `%` and bitwise operations, operands are cast to whole numbers. */ template struct ToInteger { diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index 69fa1437487..1d9c2ad7ccb 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -6,14 +6,8 @@ #include #include -#include "DataTypes/Native.h" #include "config.h" -#if USE_EMBEDDED_COMPILER -# include -# include -#endif - namespace DB { @@ -21,42 +15,8 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_DIVISION; - extern const int LOGICAL_ERROR; } -#if USE_EMBEDDED_COMPILER - -template -static llvm::Value * compileWithNullableValues(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed, F && compile_func) -{ - auto * left_type = left->getType(); - auto * right_type = right->getType(); - - if (!left_type->isStructTy() && !right_type->isStructTy()) - { - // Both arguments are not nullable. - return compile_func(b, left, right, is_signed); - } - - auto * denull_left = left_type->isStructTy() ? b.CreateExtractValue(left, {1}) : left; - auto * denull_right = right_type->isStructTy() ? b.CreateExtractValue(right, {1}) : right; - auto * denull_result = compile_func(b, denull_left, denull_right, is_signed); - - auto * nullable_result_type = toNullableType(b, denull_result->getType()); - llvm::Value * nullable_result = llvm::Constant::getNullValue(nullable_result_type); - nullable_result = b.CreateInsertValue(nullable_result, denull_result, {0}); - - auto * result_is_null = b.CreateExtractValue(nullable_result, {1}); - if (left_type->isStructTy()) - result_is_null = b.CreateOr(result_is_null, b.CreateExtractValue(left, {1})); - if (right_type->isStructTy()) - result_is_null = b.CreateOr(result_is_null, b.CreateExtractValue(right, {1})); - - return b.CreateInsertValue(nullable_result, result_is_null, {1}); -} - -#endif - template inline void throwIfDivisionLeadsToFPE(A a, B b) { @@ -198,39 +158,14 @@ struct ModuloImpl } #if USE_EMBEDDED_COMPILER - static constexpr bool compilable = true; /// Ignore exceptions in LLVM IR - - static llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - return compileWithNullableValues( - b, - left, - right, - is_signed, - [](auto & b_, auto * left_, auto * right_, auto is_signed_) { return compileImpl(b_, left_, right_, is_signed_); }); - } - - static llvm::Value * compileImpl(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - if (left->getType()->isFloatingPointTy()) - return b.CreateFRem(left, right); - else if (left->getType()->isIntegerTy()) - return is_signed ? b.CreateSRem(left, right) : b.CreateURem(left, right); - else - throw Exception(ErrorCodes::LOGICAL_ERROR, "ModuloImpl compilation expected native integer or floating point type"); - } - - #endif + static constexpr bool compilable = false; /// don't know how to throw from LLVM IR +#endif }; template struct ModuloLegacyImpl : ModuloImpl { using ResultType = typename NumberTraits::ResultOfModuloLegacy::Type; - -#if USE_EMBEDDED_COMPILER - static constexpr bool compilable = false; /// moduloLegacy is only used in partition key expression -#endif }; template @@ -259,36 +194,6 @@ struct PositiveModuloImpl : ModuloImpl } return static_cast(res); } - -#if USE_EMBEDDED_COMPILER - static constexpr bool compilable = true; /// Ignore exceptions in LLVM IR - - static llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - return compileWithNullableValues( - b, - left, - right, - is_signed, - [](auto & b_, auto * left_, auto * right_, auto is_signed_) { return compileImpl(b_, left_, right_, is_signed_); }); - } - - static llvm::Value * compileImpl(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) - { - auto * result = ModuloImpl::compileImpl(b, left, right, is_signed); - if (is_signed) - { - /// If result is negative, result += abs(right). - auto * zero = llvm::Constant::getNullValue(result->getType()); - auto * is_negative = b.CreateICmpSLT(result, zero); - auto * abs_right = b.CreateSelect(b.CreateICmpSLT(right, zero), b.CreateNeg(right), right); - return b.CreateSelect(is_negative, b.CreateAdd(result, abs_right), result); - } - else - return result; - } -#endif - }; } diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 528908b6f0f..91adf73b679 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -810,7 +810,6 @@ class FunctionBinaryArithmetic : public IFunction static constexpr bool is_division = IsOperation::division; static constexpr bool is_bit_hamming_distance = IsOperation::bit_hamming_distance; static constexpr bool is_modulo = IsOperation::modulo; - static constexpr bool is_positive_modulo = IsOperation::positive_modulo; static constexpr bool is_int_div = IsOperation::int_div; static constexpr bool is_int_div_or_zero = IsOperation::int_div_or_zero; @@ -2388,105 +2387,59 @@ ColumnPtr executeStringInteger(const ColumnsWithTypeAndName & arguments, const A if (!canBeNativeType(*arguments[0]) || !canBeNativeType(*arguments[1]) || !canBeNativeType(*result_type)) return false; - auto denull_left_type = removeNullable(arguments[0]); - auto denull_right_type = removeNullable(arguments[1]); - WhichDataType data_type_lhs(denull_left_type); - WhichDataType data_type_rhs(denull_right_type); + WhichDataType data_type_lhs(arguments[0]); + WhichDataType data_type_rhs(arguments[1]); if ((data_type_lhs.isDateOrDate32() || data_type_lhs.isDateTime()) || (data_type_rhs.isDateOrDate32() || data_type_rhs.isDateTime())) return false; - return castBothTypes( - denull_left_type.get(), - denull_right_type.get(), - [&](const auto & left, const auto & right) + return castBothTypes(arguments[0].get(), arguments[1].get(), [&](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + if constexpr (!std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - if constexpr ( - !std::is_same_v && !std::is_same_v - && !std::is_same_v && !std::is_same_v) - { - using ResultDataType = typename BinaryOperationTraits::ResultDataType; - using OpSpec = Op; - - if constexpr ( - !std::is_same_v && !IsDataTypeDecimal - && !IsDataTypeDecimal && !IsDataTypeDecimal && OpSpec::compilable) - { - if constexpr (is_modulo || is_positive_modulo) - { - using LeftType = typename LeftDataType::FieldType; - using RightType = typename RightDataType::FieldType; - using PromotedType = typename NumberTraits::ResultOfModuloNativePromotion::Type; - if constexpr (std::is_arithmetic_v) - { - return true; - } - } - else - return true; - } - } - return false; - }); + using ResultDataType = typename BinaryOperationTraits::ResultDataType; + using OpSpec = Op; + if constexpr (!std::is_same_v && !IsDataTypeDecimal && OpSpec::compilable) + return true; + } + return false; + }); } llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const ValuesWithType & arguments, const DataTypePtr & result_type) const override { assert(2 == arguments.size()); - auto denull_left_type = removeNullable(arguments[0].type); - auto denull_right_type = removeNullable(arguments[1].type); llvm::Value * result = nullptr; - - castBothTypes( - denull_left_type.get(), - denull_right_type.get(), - [&](const auto & left, const auto & right) + castBothTypes(arguments[0].type.get(), arguments[1].type.get(), [&](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + if constexpr (!std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - if constexpr ( - !std::is_same_v && !std::is_same_v - && !std::is_same_v && !std::is_same_v) + using ResultDataType = typename BinaryOperationTraits::ResultDataType; + using OpSpec = Op; + if constexpr (!std::is_same_v && !IsDataTypeDecimal && OpSpec::compilable) { - using ResultDataType = typename BinaryOperationTraits::ResultDataType; - using OpSpec = Op; - if constexpr ( - !std::is_same_v && !IsDataTypeDecimal - && !IsDataTypeDecimal && !IsDataTypeDecimal && OpSpec::compilable) - { - auto & b = static_cast &>(builder); - if constexpr (is_modulo || is_positive_modulo) - { - using LeftType = typename LeftDataType::FieldType; - using RightType = typename RightDataType::FieldType; - using PromotedType = typename NumberTraits::ResultOfModuloNativePromotion::Type; - if constexpr (std::is_arithmetic_v) - { - DataTypePtr promoted_type = std::make_shared>(); - if (result_type->isNullable()) - promoted_type = std::make_shared(promoted_type); + auto & b = static_cast &>(builder); + auto * lval = nativeCast(b, arguments[0], result_type); + auto * rval = nativeCast(b, arguments[1], result_type); + result = OpSpec::compile(b, lval, rval, std::is_signed_v); - auto * lval = nativeCast(b, arguments[0], promoted_type); - auto * rval = nativeCast(b, arguments[1], promoted_type); - result = nativeCast( - b, promoted_type, OpSpec::compile(b, lval, rval, std::is_signed_v), result_type); - return true; - } - } - else - { - auto * lval = nativeCast(b, arguments[0], result_type); - auto * rval = nativeCast(b, arguments[1], result_type); - result = OpSpec::compile(b, lval, rval, std::is_signed_v); - return true; - } - } + return true; } - return false; - }); + } + + return false; + }); return result; } diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 72f5e37e99e..cf79bb06430 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -489,7 +489,9 @@ public: { using DataType = std::decay_t; if constexpr (std::is_same_v || std::is_same_v) + { return false; + } else { using T0 = typename DataType::FieldType; @@ -511,7 +513,9 @@ public: { using DataType = std::decay_t; if constexpr (std::is_same_v || std::is_same_v) + { return false; + } else { using T0 = typename DataType::FieldType; @@ -519,16 +523,8 @@ public: if constexpr (!std::is_same_v && !IsDataTypeDecimal && Op::compilable) { auto & b = static_cast &>(builder); - if constexpr (std::is_same_v, AbsImpl> || std::is_same_v, BitCountImpl>) - { - /// We don't need to cast the argument to the result type if it's abs/bitcount function. - result = Op::compile(b, arguments[0].value, is_signed_v); - } - else - { - auto * v = nativeCast(b, arguments[0], result_type); - result = Op::compile(b, v, is_signed_v); - } + auto * v = nativeCast(b, arguments[0], result_type); + result = Op::compile(b, v, is_signed_v); return true; } diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 5a4276e0d75..d8ff9b1699e 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1,21 +1,17 @@ #pragma once -// Include this first, because `#define _asan_poison_address` from -// llvm/Support/Compiler.h conflicts with its forward declaration in -// sanitizer/asan_interface.h -#include -#include -#include +#include +#include +#include -#include +#include #include #include -#include #include +#include #include -#include -#include -#include +#include + #include #include #include @@ -28,23 +24,22 @@ #include #include #include -#include + +#include +#include + #include +#include #include + +#include +#include + #include #include -#include -#include -#include -#include -#include -#include "DataTypes/NumberTraits.h" -#if USE_EMBEDDED_COMPILER -# include -# include -# include -#endif +#include +#include namespace DB { @@ -635,61 +630,6 @@ struct GenericComparisonImpl } }; - -#if USE_EMBEDDED_COMPILER - -template