From 318e9b4f33b3eb0424d6fcc5b435c4e4dd531ad4 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Thu, 21 Nov 2024 03:29:07 +0000 Subject: [PATCH] add plain_object_storage_write_fail_on_directory_move fault injection --- src/Common/FailPoint.cpp | 1 + ...torageFromPlainObjectStorageOperations.cpp | 8 ++--- ...aStorageFromPlainObjectStorageOperations.h | 1 - .../03008_s3_plain_rewritable_fault.reference | 13 ++++++- .../03008_s3_plain_rewritable_fault.sh | 34 ++++++++++++++----- 5 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index a640523ac52..027cc347386 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -78,6 +78,7 @@ static struct InitFiu REGULAR(zero_copy_lock_zk_fail_before_op) \ REGULAR(zero_copy_lock_zk_fail_after_op) \ REGULAR(plain_object_storage_write_fail_on_directory_create) \ + REGULAR(plain_object_storage_write_fail_on_directory_move) \ namespace FailPoints diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp index f2b9e5161b8..96832c4dfb1 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp @@ -25,6 +25,7 @@ extern const int FAULT_INJECTED; namespace FailPoints { extern const char plain_object_storage_write_fail_on_directory_create[]; +extern const char plain_object_storage_write_fail_on_directory_move[]; } namespace @@ -190,8 +191,10 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u getLogger("MetadataStorageFromPlainObjectStorageMoveDirectoryOperation"), "Moving directory '{}' to '{}'", path_from, path_to); auto write_buf = createWriteBuf(path_from, path_to, /* validate_content */ true); - write_created = true; writeString(path_to.string(), *write_buf); + fiu_do_on(FailPoints::plain_object_storage_write_fail_on_directory_move, { + throw Exception(ErrorCodes::FAULT_INJECTED, "Injecting fault when moving from '{}' to '{}'", path_from, path_to); + }); write_buf->finalize(); /// parent_path() removes the trailing '/'. @@ -216,10 +219,7 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq std::lock_guard lock(path_map.mutex); auto & map = path_map.map; map.emplace(path_from.parent_path(), map.extract(path_to.parent_path()).mapped()); - } - if (write_created) - { auto write_buf = createWriteBuf(path_to, path_from, /* verify_content */ false); writeString(path_from.string(), *write_buf); write_buf->finalize(); diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h index 0a26dfed3e2..393fe21a4c3 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h +++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h @@ -42,7 +42,6 @@ private: ObjectStoragePtr object_storage; const std::string metadata_key_prefix; - bool write_created = false; bool write_finalized = false; std::unique_ptr diff --git a/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.reference b/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.reference index 573541ac970..2f87357245f 100644 --- a/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.reference +++ b/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.reference @@ -1 +1,12 @@ -0 +1 2 +2 2 +3 1 +4 7 +5 10 +6 12 +1 2 +2 2 +3 1 +4 7 +5 10 +6 12 diff --git a/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.sh b/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.sh index 274e7c148c2..89ed24e2b43 100755 --- a/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.sh +++ b/tests/queries/0_stateless/03008_s3_plain_rewritable_fault.sh @@ -1,15 +1,17 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-shared-merge-tree +# Tags: no-fasttest, no-shared-merge-tree, no-parallel # Tag no-fasttest: requires S3 # Tag no-shared-merge-tree: does not support replication +# Tag no-parallel: uses failpoints CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh on_exit() { - ${CLICKHOUSE_CLIENT} --query " - SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create + ${CLICKHOUSE_CLIENT} -m --query " + SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create; + SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_move; " } @@ -27,17 +29,33 @@ SETTINGS disk = disk( secret_access_key = clickhouse); " +${CLICKHOUSE_CLIENT} --query " +INSERT INTO test_s3_mt_fault (*) VALUES (1, 2), (2, 2), (3, 1), (4, 7), (5, 10), (6, 12); +OPTIMIZE TABLE test_s3_mt_fault FINAL; +" + ${CLICKHOUSE_CLIENT} --query " SYSTEM ENABLE FAILPOINT plain_object_storage_write_fail_on_directory_create " ${CLICKHOUSE_CLIENT} --query " -INSERT INTO test_s3_mt_fault (*) select number, number from numbers_mt(100)'" 2>&1 | grep -Fq "FAULT_INJECTED" +INSERT INTO test_s3_mt_fault (*) select number, number from numbers_mt(100)" 2>&1 | grep -Fq "FAULT_INJECTED" -${CLICKHOUSE_CLIENT} --query "SELECT count(*) from test_s3_mt_fault;" - -${CLICKHOUSE_CLIENT} --query "DROP TABLE test_s3_mt_fault;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_s3_mt_fault;" ${CLICKHOUSE_CLIENT} --query " -SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create +SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create; +SYSTEM ENABLE FAILPOINT plain_object_storage_write_fail_on_directory_move; " + +${CLICKHOUSE_CLIENT} --query " +INSERT INTO test_s3_mt_fault (*) select number, number from numbers_mt(100); +" 2>&1 | grep -Fq "FAULT_INJECTED" + +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_s3_mt_fault;" + +${CLICKHOUSE_CLIENT} --query " +SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_move; +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE test_s3_mt_fault"