retry DROP after restart

dbms/src/Databases/DatabaseAtomic.cpp

@@ -3,6 +3,7 @@
 #include <Poco/File.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Common/Stopwatch.h>
 
 
 namespace DB
@@ -80,7 +81,7 @@ void DatabaseAtomic::dropTable(const Context & context, const String & table_nam
         // 1. CREATE table_name: + table_name.sql
         // 2. DROP table_name: table_name.sql -> table_name.sql.tmp_drop
         // 3. CREATE table_name: + table_name.sql
-        // 4. DROP table_name: table_name.sql -> table_name.sql.tmp_drop overwrites table_name.sql.tmp_drop ?
+        // 4. DROP table_name: table_name.sql -> table_name.sql.tmp_drop overwrites table_name.sql.tmp_drop
         Poco::File(table_metadata_path).renameTo(table_metadata_path_drop);
 
         {
@@ -127,6 +128,27 @@ void DatabaseAtomic::renameTable(const Context & context, const String & table_n
 
 void DatabaseAtomic::loadStoredObjects(Context & context, bool has_force_restore_data_flag)
 {
+    iterateMetadataFiles(context, [](const String &) {}, [&](const String & file_name)
+    {
+        String full_path = getMetadataPath() + file_name;
+        LOG_INFO(log, "Trying load partially dropped table from " << full_path);
+        try
+        {
+            auto ast = parseQueryFromMetadata(context, full_path, /*throw_on_error*/ true, /*remove_empty*/false);
+            if (!ast) //TODO why?
+                return;
+            auto & query = ast->as<ASTCreateQuery &>();
+            auto [_, table] = createTableFromAST(query, database_name, getTableDataPath(query), context, has_force_restore_data_flag);
+            time_t drop_time = Poco::File(full_path).getLastModified().epochTime();
+            tables_to_drop.push_back({table, context.getPath() + getTableDataPath(query), drop_time});
+        }
+        catch (Exception & e)
+        {
+            e.addMessage("Cannot complete table drop " + full_path);
+            //TODO may be remove data dir (if UUID successfully parsed) and .tmp_drop metadata?
+        }
+    });
+
     DatabaseOrdinary::loadStoredObjects(context, has_force_restore_data_flag);
     drop_task->activateAndSchedule();
 }

dbms/src/Databases/DatabaseOnDisk.cpp

@@ -342,7 +342,33 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_na
         return static_cast<time_t>(0);
 }
 
-void DatabaseOnDisk::iterateMetadataFiles(const Context & /*context*/, const IteratingFunction & iterating_function) const
+void DatabaseOnDisk::iterateMetadataFiles(const Context & context,
+                                          const DatabaseOnDisk::IteratingFunction & process_metadata_file) const
+{
+    IteratingFunction process_tmp_drop_metadata_file = [&](const String & file_name)
+    {
+        static const char * tmp_drop_ext = ".sql.tmp_drop";
+        const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext));
+        if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
+        {
+            Poco::File(getMetadataPath() + file_name).renameTo(context.getPath() + getMetadataPath() + object_name + ".sql");
+            LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
+            process_metadata_file(object_name + ".sql");
+        }
+        else
+        {
+            LOG_INFO(log, "Removing file " << getMetadataPath() + file_name);
+            Poco::File(getMetadataPath() + file_name).remove();
+        }
+    };
+
+    IteratingFunction do_nothing = [](const String &){};
+    //FIXME refactor this trash
+    iterateMetadataFiles(context, process_metadata_file, dynamic_cast<const DatabaseAtomic *>(this) ? do_nothing : process_tmp_drop_metadata_file);
+}
+
+void DatabaseOnDisk::iterateMetadataFiles(const Context & /*context*/, const IteratingFunction & process_metadata_file,
+                                          const IteratingFunction & process_tmp_drop_metadata_file) const
 {
     Poco::DirectoryIterator dir_end;
     for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
@@ -355,39 +381,22 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & /*context*/, const Ite
         if (endsWith(dir_it.name(), ".sql.bak"))
             continue;
 
-        // There are files that we tried to delete previously
         static const char * tmp_drop_ext = ".sql.tmp_drop";
         if (endsWith(dir_it.name(), tmp_drop_ext))
         {
-            //const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
+            /// There are files that we tried to delete previously
-            //if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
+            process_tmp_drop_metadata_file(dir_it.name());
-            //{
-            //    /// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
-            //      //TODO check all paths
-            //    Poco::File(dir_it->path()).renameTo(context.getPath() + getMetadataPath() + object_name + ".sql");
-            //    LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
-            //    iterating_function(object_name + ".sql");
-            //}
-            //else
-            //{
-            //    LOG_INFO(log, "Removing file " << dir_it->path());
-            //    Poco::File(dir_it->path()).remove();
-            //}
-            continue;
         }
-
+        else if (endsWith(dir_it.name(), ".sql.tmp"))
-        /// There are files .sql.tmp - delete
-        if (endsWith(dir_it.name(), ".sql.tmp"))
         {
+            /// There are files .sql.tmp - delete
             LOG_INFO(log, "Removing file " << dir_it->path());
             Poco::File(dir_it->path()).remove();
-            continue;
         }
-
+        else if (endsWith(dir_it.name(), ".sql"))
-        /// The required files have names like `table_name.sql`
-        if (endsWith(dir_it.name(), ".sql"))
         {
-            iterating_function(dir_it.name());
+            /// The required files have names like `table_name.sql`
+            process_metadata_file(dir_it.name());
         }
         else
             throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
@@ -437,7 +446,11 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const Context & context, const Str
     auto & create = ast->as<ASTCreateQuery &>();
     if (create.uuid != UUIDHelpers::Nil)
     {
+        //FIXME it can be .sql or .sql.tmp_drop
         String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName();
+        if (Poco::Path(table_name).makeFile().getExtension() == "sql")
+            table_name = Poco::Path(table_name).makeFile().getBaseName();
+
         if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER)
             LOG_WARNING(log, "File " << metadata_file_path << " contains both UUID and table name. "
                                                     "Will use name `" << table_name << "` instead of `" << create.table << "`");

dbms/src/Databases/DatabaseOnDisk.h

@@ -66,7 +66,10 @@ protected:
     static constexpr const char * drop_suffix = ".tmp_drop";
 
     using IteratingFunction = std::function<void(const String &)>;
-    void iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const;
+
+    void iterateMetadataFiles(const Context & context, const IteratingFunction & process_metadata_file) const;
+    void iterateMetadataFiles(const Context & context, const IteratingFunction & process_metadata_file,
+                              const IteratingFunction & process_tmp_drop_metadata_file) const;
 
     ASTPtr getCreateTableQueryImpl(
         const Context & context,
@@ -76,6 +79,7 @@ protected:
     ASTPtr parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const;
     ASTPtr getCreateQueryFromMetadata(const Context & context, const String & metadata_path, bool throw_on_error) const;
 
+
     //bool detachTableAndRemoveMetadata(const String & table_name);
     //void replaceMetadata(const ASTPtr & create, );
 

dbms/src/Databases/DatabaseOrdinary.cpp

@@ -29,6 +29,7 @@
 #include <Common/typeid_cast.h>
 #include <common/logger_useful.h>
 #include <ext/scope_guard.h>
+#include "DatabaseAtomic.h"
 
 
 namespace DB

dbms/src/Databases/DatabaseOrdinary.h

@@ -27,7 +27,7 @@ public:
         const String & name,
         const StorageInMemoryMetadata & metadata) override;
 
-private:
+protected:
 
     void startupTables(ThreadPool & thread_pool);
 };