reduce copy-paste

dbms/src/Databases/DatabaseLazy.cpp

@@ -74,8 +74,8 @@ void DatabaseLazy::loadTables(
     Context & /* context */,
     bool /* has_force_restore_data_flag */)
 {
-    iterateTableFiles([this](const Poco::DirectoryIterator & dir_it) {
+    DatabaseOnDisk::iterateTableFiles(*this, log, [this](const String & file_name) {
-        const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - 4);
+        const std::string table_name = file_name.substr(0, file_name.size() - 4);
         attachTable(table_name, nullptr);
     });
 }
@@ -291,56 +291,6 @@ String DatabaseLazy::getTableMetadataPath(const String & table_name) const
     return DatabaseOnDisk::getTableMetadataPath(*this, table_name);
 }
 
-void DatabaseLazy::iterateTableFiles(const IteratingFunction & iterating_function) const
-{
-    Poco::DirectoryIterator dir_end;
-    for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
-    {
-        /// For '.svn', '.gitignore' directory and similar.
-        if (dir_it.name().at(0) == '.')
-            continue;
-
-        /// There are .sql.bak files - skip them.
-        if (endsWith(dir_it.name(), ".sql.bak"))
-            continue;
-
-        // There are files that we tried to delete previously
-        static const char * tmp_drop_ext = ".sql.tmp_drop";
-        if (endsWith(dir_it.name(), tmp_drop_ext))
-        {
-            const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
-            if (Poco::File(getDataPath() + '/' + table_name).exists())
-            {
-                Poco::File(dir_it->path()).renameTo(table_name + ".sql");
-                LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
-            }
-            else
-            {
-                LOG_INFO(log, "Removing file " << dir_it->path());
-                Poco::File(dir_it->path()).remove();
-            }
-            continue;
-        }
-
-        /// There are files .sql.tmp - delete
-        if (endsWith(dir_it.name(), ".sql.tmp"))
-        {
-            LOG_INFO(log, "Removing file " << dir_it->path());
-            Poco::File(dir_it->path()).remove();
-            continue;
-        }
-
-        /// The required files have names like `table_name.sql`
-        if (endsWith(dir_it.name(), ".sql"))
-        {
-            iterating_function(dir_it);
-        }
-        else
-            throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + metadata_path,
-                ErrorCodes::INCORRECT_FILE_NAME);
-    }
-}
-
 StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table_name) const
 {
     clearExpiredTables();

dbms/src/Databases/DatabaseLazy.h

@@ -142,9 +142,6 @@ private:
 
     Poco::Logger * log;
     
-    using IteratingFunction = std::function<void(const Poco::DirectoryIterator &)>;
-    void iterateTableFiles(const IteratingFunction & iterating_function) const;
-    
     StoragePtr loadTable(const Context & context, const String & table_name) const;
 
     void clearExpiredTables() const;

dbms/src/Databases/DatabaseOnDisk.cpp

@@ -22,11 +22,12 @@ namespace DB
 
 namespace ErrorCodes
 {
+    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int FILE_DOESNT_EXIST;
+    extern const int INCORRECT_FILE_NAME;
+    extern const int SYNTAX_ERROR;
     extern const int TABLE_ALREADY_EXISTS;
     extern const int UNKNOWN_TABLE;
-    extern const int FILE_DOESNT_EXIST;
-    extern const int CANNOT_GET_CREATE_TABLE_QUERY;
-    extern const int SYNTAX_ERROR;
 }
 
 
@@ -232,4 +233,54 @@ String DatabaseOnDisk::getTableMetadataPath(const IDatabase & database, const St
     return detail::getTableMetadataPath(database.getMetadataPath(), table_name);
 }
 
+void DatabaseOnDisk::iterateTableFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function)
+{
+    Poco::DirectoryIterator dir_end;
+    for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it)
+    {
+        /// For '.svn', '.gitignore' directory and similar.
+        if (dir_it.name().at(0) == '.')
+            continue;
+
+        /// There are .sql.bak files - skip them.
+        if (endsWith(dir_it.name(), ".sql.bak"))
+            continue;
+
+        // There are files that we tried to delete previously
+        static const char * tmp_drop_ext = ".sql.tmp_drop";
+        if (endsWith(dir_it.name(), tmp_drop_ext))
+        {
+            const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
+            if (Poco::File(database.getDataPath() + '/' + table_name).exists())
+            {
+                Poco::File(dir_it->path()).renameTo(table_name + ".sql");
+                LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
+            }
+            else
+            {
+                LOG_INFO(log, "Removing file " << dir_it->path());
+                Poco::File(dir_it->path()).remove();
+            }
+            continue;
+        }
+
+        /// There are files .sql.tmp - delete
+        if (endsWith(dir_it.name(), ".sql.tmp"))
+        {
+            LOG_INFO(log, "Removing file " << dir_it->path());
+            Poco::File(dir_it->path()).remove();
+            continue;
+        }
+
+        /// The required files have names like `table_name.sql`
+        if (endsWith(dir_it.name(), ".sql"))
+        {
+            iterating_function(dir_it.name());
+        }
+        else
+            throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + database.getMetadataPath(),
+                ErrorCodes::INCORRECT_FILE_NAME);
+    }
+}
+
 }

dbms/src/Databases/DatabaseOnDisk.h

@@ -64,6 +64,9 @@ public:
         const IDatabase & database,
         const String & table_name);
 
+    using IteratingFunction = std::function<void(const String &)>;
+    static void iterateTableFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function);
+
 private:
     static ASTPtr getCreateTableQueryImpl(
         const IDatabase & database,

dbms/src/Databases/DatabaseOrdinary.cpp

@@ -47,12 +47,6 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_TABLES = 256;
 static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
 static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
 
-namespace detail
-{
-    extern String getTableMetadataPath(const String & base_path, const String & table_name);
-    extern String getDatabaseMetadataPath(const String & base_path);
-}
-
 static void loadTable(
     Context & context,
     const String & database_metadata_path,
@@ -117,50 +111,9 @@ void DatabaseOrdinary::loadTables(
     using FileNames = std::vector<std::string>;
     FileNames file_names;
 
-    Poco::DirectoryIterator dir_end;
+    DatabaseOnDisk::iterateTableFiles(*this, log, [&file_names](const String & file_name) {
-    for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
+        file_names.push_back(file_name);
-    {
+    });
-        /// For '.svn', '.gitignore' directory and similar.
-        if (dir_it.name().at(0) == '.')
-            continue;
-
-        /// There are .sql.bak files - skip them.
-        if (endsWith(dir_it.name(), ".sql.bak"))
-            continue;
-
-        // There are files that we tried to delete previously
-        static const char * tmp_drop_ext = ".sql.tmp_drop";
-        if (endsWith(dir_it.name(), tmp_drop_ext))
-        {
-            const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
-            if (Poco::File(getDataPath() + '/' + table_name).exists())
-            {
-                Poco::File(dir_it->path()).renameTo(table_name + ".sql");
-                LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
-            }
-            else
-            {
-                LOG_INFO(log, "Removing file " << dir_it->path());
-                Poco::File(dir_it->path()).remove();
-            }
-            continue;
-        }
-
-        /// There are files .sql.tmp - delete
-        if (endsWith(dir_it.name(), ".sql.tmp"))
-        {
-            LOG_INFO(log, "Removing file " << dir_it->path());
-            Poco::File(dir_it->path()).remove();
-            continue;
-        }
-
-        /// The required files have names like `table_name.sql`
-        if (endsWith(dir_it.name(), ".sql"))
-            file_names.push_back(dir_it.name());
-        else
-            throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + metadata_path,
-                ErrorCodes::INCORRECT_FILE_NAME);
-    }
 
     if (file_names.empty())
         return;
@@ -242,128 +195,17 @@ void DatabaseOrdinary::createTable(
     const StoragePtr & table,
     const ASTPtr & query)
 {
-    const auto & settings = context.getSettingsRef();
+    DatabaseOnDisk::createTable(*this, context, table_name, table, query);
-
-    /// Create a file with metadata if necessary - if the query is not ATTACH.
-    /// Write the query of `ATTACH table` to it.
-
-    /** The code is based on the assumption that all threads share the same order of operations
-      * - creating the .sql.tmp file;
-      * - adding a table to `tables`;
-      * - rename .sql.tmp to .sql.
-      */
-
-    /// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH.
-    /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
-
-    if (isTableExist(context, table_name))
-        throw Exception("Table " + getDatabaseName() + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
-
-    String table_metadata_path = getTableMetadataPath(table_name);
-    String table_metadata_tmp_path = table_metadata_path + ".tmp";
-    String statement;
-
-    {
-        statement = getTableDefinitionFromCreateQuery(query);
-
-        /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
-        WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
-        writeString(statement, out);
-        out.next();
-        if (settings.fsync_metadata)
-            out.sync();
-        out.close();
-    }
-
-    try
-    {
-        /// Add a table to the map of known tables.
-        attachTable(table_name, table);
-
-        /// If it was ATTACH query and file with table metadata already exist
-        /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
-        Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
-    }
-    catch (...)
-    {
-        Poco::File(table_metadata_tmp_path).remove();
-        throw;
-    }
 }
 
 
 void DatabaseOrdinary::removeTable(
-    const Context & /*context*/,
+    const Context & context,
     const String & table_name)
 {
-    StoragePtr res = detachTable(table_name);
+    DatabaseOnDisk::removeTable(*this, context, table_name, log);
-
-    String table_metadata_path = getTableMetadataPath(table_name);
-
-    try
-    {
-        Poco::File(table_metadata_path).remove();
-    }
-    catch (...)
-    {
-        try
-        {
-            Poco::File(table_metadata_path + ".tmp_drop").remove();
-            return;
-        }
-        catch (...)
-        {
-            LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
-        }
-        attachTable(table_name, res);
-        throw;
-    }
 }
 
-static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true)
-{
-    String query;
-
-    try
-    {
-        ReadBufferFromFile in(metadata_path, 4096);
-        readStringUntilEOF(query, in);
-    }
-    catch (const Exception & e)
-    {
-        if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
-            return nullptr;
-        else
-            throw;
-    }
-
-    ParserCreateQuery parser;
-    const char * pos = query.data();
-    std::string error_message;
-    auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
-                             "in file " + metadata_path, /* allow_multi_statements = */ false, 0);
-
-    if (!ast && throw_on_error)
-        throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
-
-    return ast;
-}
-
-static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error)
-{
-    ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error);
-
-    if (ast)
-    {
-        auto & ast_create_query = ast->as<ASTCreateQuery &>();
-        ast_create_query.attach = false;
-        ast_create_query.database = database;
-    }
-
-    return ast;
-}
-
-
 void DatabaseOrdinary::renameTable(
     const Context & context,
     const String & table_name,
@@ -371,41 +213,7 @@ void DatabaseOrdinary::renameTable(
     const String & to_table_name,
     TableStructureWriteLockHolder & lock)
 {
-    DatabaseOrdinary * to_database_concrete = typeid_cast<DatabaseOrdinary *>(&to_database);
+    DatabaseOnDisk::renameTable<DatabaseOrdinary>(*this, context, table_name, to_database, to_table_name, lock);
-
-    if (!to_database_concrete)
-        throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
-
-    StoragePtr table = tryGetTable(context, table_name);
-
-    if (!table)
-        throw Exception("Table " + getDatabaseName() + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
-
-    /// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
-    try
-    {
-        table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/",
-            to_database_concrete->getDatabaseName(),
-            to_table_name, lock);
-    }
-    catch (const Exception &)
-    {
-        throw;
-    }
-    catch (const Poco::Exception & e)
-    {
-        /// Better diagnostics.
-        throw Exception{Exception::CreateFromPoco, e};
-    }
-
-    ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(getMetadataPath(), table_name));
-    if (!ast)
-        throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST);
-    ast->as<ASTCreateQuery &>().table = to_table_name;
-
-    /// NOTE Non-atomic.
-    to_database_concrete->createTable(context, to_table_name, table, ast);
-    removeTable(context, table_name);
 }
 
 
@@ -426,53 +234,19 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime(
     }
 }
 
-ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context,
-                                                 const String & table_name, bool throw_on_error) const
-{
-    ASTPtr ast;
-
-    auto table_metadata_path = detail::getTableMetadataPath(getMetadataPath(), table_name);
-    ast = getCreateQueryFromMetadata(table_metadata_path, getDatabaseName(), throw_on_error);
-    if (!ast && throw_on_error)
-    {
-        /// Handle system.* tables for which there are no table.sql files.
-        bool has_table = tryGetTable(context, table_name) != nullptr;
-
-        auto msg = has_table
-                   ? "There is no CREATE TABLE query for table "
-                   : "There is no metadata file for table ";
-
-        throw Exception(msg + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
-    }
-
-    return ast;
-}
-
 ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
 {
-    return getCreateTableQueryImpl(context, table_name, true);
+    return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
 }
 
 ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
 {
-    return getCreateTableQueryImpl(context, table_name, false);
+    return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
 }
 
-ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const
+ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & context) const
 {
-    ASTPtr ast;
+    return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
-
-    auto database_metadata_path = detail::getDatabaseMetadataPath(getMetadataPath());
-    ast = getCreateQueryFromMetadata(database_metadata_path, getDatabaseName(), true);
-    if (!ast)
-    {
-        /// Handle databases (such as default) for which there are no database.sql files.
-        String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
-        ParserCreateQuery parser;
-        ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
-    }
-
-    return ast;
 }
 
 void DatabaseOrdinary::alterTable(
@@ -546,8 +320,7 @@ void DatabaseOrdinary::alterTable(
 
 void DatabaseOrdinary::drop()
 {
-    Poco::File(getDataPath()).remove(false);
+    DatabaseOnDisk::drop(*this);
-    Poco::File(getMetadataPath()).remove(false);
 }