add plain_object_storage_write_fail_on_directory_move fault injection

src/Common/FailPoint.cpp

@@ -78,6 +78,7 @@ static struct InitFiu
     REGULAR(zero_copy_lock_zk_fail_before_op) \
     REGULAR(zero_copy_lock_zk_fail_after_op) \
     REGULAR(plain_object_storage_write_fail_on_directory_create) \
+    REGULAR(plain_object_storage_write_fail_on_directory_move) \
 
 
 namespace FailPoints

src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp

@@ -25,6 +25,7 @@ extern const int FAULT_INJECTED;
 namespace FailPoints
 {
 extern const char plain_object_storage_write_fail_on_directory_create[];
+extern const char plain_object_storage_write_fail_on_directory_move[];
 }
 
 namespace
@@ -190,8 +191,10 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
         getLogger("MetadataStorageFromPlainObjectStorageMoveDirectoryOperation"), "Moving directory '{}' to '{}'", path_from, path_to);
 
     auto write_buf = createWriteBuf(path_from, path_to, /* validate_content */ true);
-    write_created = true;
     writeString(path_to.string(), *write_buf);
+    fiu_do_on(FailPoints::plain_object_storage_write_fail_on_directory_move, {
+        throw Exception(ErrorCodes::FAULT_INJECTED, "Injecting fault when moving from '{}' to '{}'", path_from, path_to);
+    });
     write_buf->finalize();
 
     /// parent_path() removes the trailing '/'.
@@ -216,10 +219,7 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
         std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
         map.emplace(path_from.parent_path(), map.extract(path_to.parent_path()).mapped());
-    }
 
-    if (write_created)
-    {
         auto write_buf = createWriteBuf(path_to, path_from, /* verify_content */ false);
         writeString(path_from.string(), *write_buf);
         write_buf->finalize();

src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h

@@ -42,7 +42,6 @@ private:
     ObjectStoragePtr object_storage;
     const std::string metadata_key_prefix;
 
-    bool write_created = false;
     bool write_finalized = false;
 
     std::unique_ptr<WriteBufferFromFileBase>

tests/queries/0_stateless/03008_s3_plain_rewritable_fault.reference

@@ -1 +1,12 @@
-0
+1	2
+2	2
+3	1
+4	7
+5	10
+6	12
+1	2
+2	2
+3	1
+4	7
+5	10
+6	12

tests/queries/0_stateless/03008_s3_plain_rewritable_fault.sh

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-shared-merge-tree
+# Tags: no-fasttest, no-shared-merge-tree, no-parallel
 # Tag no-fasttest: requires S3
 # Tag no-shared-merge-tree: does not support replication
+# Tag no-parallel: uses failpoints
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
 on_exit() {
-    ${CLICKHOUSE_CLIENT} --query "
+    ${CLICKHOUSE_CLIENT} -m --query "
-    SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create
+    SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create;
+    SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_move;
 "
 }
 
@@ -27,17 +29,33 @@ SETTINGS disk = disk(
     secret_access_key = clickhouse);
 "
 
+${CLICKHOUSE_CLIENT} --query "
+INSERT INTO test_s3_mt_fault (*) VALUES (1, 2), (2, 2), (3, 1), (4, 7), (5, 10), (6, 12);
+OPTIMIZE TABLE test_s3_mt_fault FINAL;
+"
+
 ${CLICKHOUSE_CLIENT} --query "
 SYSTEM ENABLE FAILPOINT plain_object_storage_write_fail_on_directory_create
 "
 
 ${CLICKHOUSE_CLIENT} --query "
-INSERT INTO test_s3_mt_fault (*) select number, number from numbers_mt(100)'" 2>&1 | grep -Fq "FAULT_INJECTED"
+INSERT INTO test_s3_mt_fault (*) select number, number from numbers_mt(100)" 2>&1 | grep -Fq "FAULT_INJECTED"
 
-${CLICKHOUSE_CLIENT} --query "SELECT count(*) from test_s3_mt_fault;"
+${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_s3_mt_fault;"
-
-${CLICKHOUSE_CLIENT} --query "DROP TABLE test_s3_mt_fault;"
 
 ${CLICKHOUSE_CLIENT} --query "
-SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create
+SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create;
+SYSTEM ENABLE FAILPOINT plain_object_storage_write_fail_on_directory_move;
 "
+
+${CLICKHOUSE_CLIENT} --query "
+INSERT INTO test_s3_mt_fault (*) select number, number from numbers_mt(100);
+" 2>&1 | grep -Fq "FAULT_INJECTED"
+
+${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_s3_mt_fault;"
+
+${CLICKHOUSE_CLIENT} --query "
+SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_move;
+"
+
+${CLICKHOUSE_CLIENT} --query "DROP TABLE test_s3_mt_fault"