From 7ac9fadc0d7db9e590dd0bd848a2dcd6e54f74b2 Mon Sep 17 00:00:00 2001 From: Vasilyev Nikita Date: Thu, 3 Oct 2019 11:27:43 +0300 Subject: [PATCH] reduce copy-paste --- dbms/src/Databases/DatabaseLazy.cpp | 54 +---- dbms/src/Databases/DatabaseLazy.h | 3 - dbms/src/Databases/DatabaseOnDisk.cpp | 57 +++++- dbms/src/Databases/DatabaseOnDisk.h | 3 + dbms/src/Databases/DatabaseOrdinary.cpp | 251 ++---------------------- 5 files changed, 71 insertions(+), 297 deletions(-) diff --git a/dbms/src/Databases/DatabaseLazy.cpp b/dbms/src/Databases/DatabaseLazy.cpp index cca2b1da4b8..4c9e07e89d4 100644 --- a/dbms/src/Databases/DatabaseLazy.cpp +++ b/dbms/src/Databases/DatabaseLazy.cpp @@ -74,8 +74,8 @@ void DatabaseLazy::loadTables( Context & /* context */, bool /* has_force_restore_data_flag */) { - iterateTableFiles([this](const Poco::DirectoryIterator & dir_it) { - const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - 4); + DatabaseOnDisk::iterateTableFiles(*this, log, [this](const String & file_name) { + const std::string table_name = file_name.substr(0, file_name.size() - 4); attachTable(table_name, nullptr); }); } @@ -291,56 +291,6 @@ String DatabaseLazy::getTableMetadataPath(const String & table_name) const return DatabaseOnDisk::getTableMetadataPath(*this, table_name); } -void DatabaseLazy::iterateTableFiles(const IteratingFunction & iterating_function) const -{ - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) - { - /// For '.svn', '.gitignore' directory and similar. - if (dir_it.name().at(0) == '.') - continue; - - /// There are .sql.bak files - skip them. - if (endsWith(dir_it.name(), ".sql.bak")) - continue; - - // There are files that we tried to delete previously - static const char * tmp_drop_ext = ".sql.tmp_drop"; - if (endsWith(dir_it.name(), tmp_drop_ext)) - { - const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext)); - if (Poco::File(getDataPath() + '/' + table_name).exists()) - { - Poco::File(dir_it->path()).renameTo(table_name + ".sql"); - LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously"); - } - else - { - LOG_INFO(log, "Removing file " << dir_it->path()); - Poco::File(dir_it->path()).remove(); - } - continue; - } - - /// There are files .sql.tmp - delete - if (endsWith(dir_it.name(), ".sql.tmp")) - { - LOG_INFO(log, "Removing file " << dir_it->path()); - Poco::File(dir_it->path()).remove(); - continue; - } - - /// The required files have names like `table_name.sql` - if (endsWith(dir_it.name(), ".sql")) - { - iterating_function(dir_it); - } - else - throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + metadata_path, - ErrorCodes::INCORRECT_FILE_NAME); - } -} - StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table_name) const { clearExpiredTables(); diff --git a/dbms/src/Databases/DatabaseLazy.h b/dbms/src/Databases/DatabaseLazy.h index 1df9eef6efc..5110d584170 100644 --- a/dbms/src/Databases/DatabaseLazy.h +++ b/dbms/src/Databases/DatabaseLazy.h @@ -142,9 +142,6 @@ private: Poco::Logger * log; - using IteratingFunction = std::function; - void iterateTableFiles(const IteratingFunction & iterating_function) const; - StoragePtr loadTable(const Context & context, const String & table_name) const; void clearExpiredTables() const; diff --git a/dbms/src/Databases/DatabaseOnDisk.cpp b/dbms/src/Databases/DatabaseOnDisk.cpp index 65f47d263e2..aa4135a4186 100644 --- a/dbms/src/Databases/DatabaseOnDisk.cpp +++ b/dbms/src/Databases/DatabaseOnDisk.cpp @@ -22,11 +22,12 @@ namespace DB namespace ErrorCodes { + extern const int CANNOT_GET_CREATE_TABLE_QUERY; + extern const int FILE_DOESNT_EXIST; + extern const int INCORRECT_FILE_NAME; + extern const int SYNTAX_ERROR; extern const int TABLE_ALREADY_EXISTS; extern const int UNKNOWN_TABLE; - extern const int FILE_DOESNT_EXIST; - extern const int CANNOT_GET_CREATE_TABLE_QUERY; - extern const int SYNTAX_ERROR; } @@ -232,4 +233,54 @@ String DatabaseOnDisk::getTableMetadataPath(const IDatabase & database, const St return detail::getTableMetadataPath(database.getMetadataPath(), table_name); } +void DatabaseOnDisk::iterateTableFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function) +{ + Poco::DirectoryIterator dir_end; + for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it) + { + /// For '.svn', '.gitignore' directory and similar. + if (dir_it.name().at(0) == '.') + continue; + + /// There are .sql.bak files - skip them. + if (endsWith(dir_it.name(), ".sql.bak")) + continue; + + // There are files that we tried to delete previously + static const char * tmp_drop_ext = ".sql.tmp_drop"; + if (endsWith(dir_it.name(), tmp_drop_ext)) + { + const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext)); + if (Poco::File(database.getDataPath() + '/' + table_name).exists()) + { + Poco::File(dir_it->path()).renameTo(table_name + ".sql"); + LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously"); + } + else + { + LOG_INFO(log, "Removing file " << dir_it->path()); + Poco::File(dir_it->path()).remove(); + } + continue; + } + + /// There are files .sql.tmp - delete + if (endsWith(dir_it.name(), ".sql.tmp")) + { + LOG_INFO(log, "Removing file " << dir_it->path()); + Poco::File(dir_it->path()).remove(); + continue; + } + + /// The required files have names like `table_name.sql` + if (endsWith(dir_it.name(), ".sql")) + { + iterating_function(dir_it.name()); + } + else + throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + database.getMetadataPath(), + ErrorCodes::INCORRECT_FILE_NAME); + } +} + } \ No newline at end of file diff --git a/dbms/src/Databases/DatabaseOnDisk.h b/dbms/src/Databases/DatabaseOnDisk.h index 9df77e34d57..02b9c7aefe5 100644 --- a/dbms/src/Databases/DatabaseOnDisk.h +++ b/dbms/src/Databases/DatabaseOnDisk.h @@ -64,6 +64,9 @@ public: const IDatabase & database, const String & table_name); + using IteratingFunction = std::function; + static void iterateTableFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function); + private: static ASTPtr getCreateTableQueryImpl( const IDatabase & database, diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 3f48c30900c..e034cfb5088 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -47,12 +47,6 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_TABLES = 256; static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5; static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; -namespace detail -{ - extern String getTableMetadataPath(const String & base_path, const String & table_name); - extern String getDatabaseMetadataPath(const String & base_path); -} - static void loadTable( Context & context, const String & database_metadata_path, @@ -117,50 +111,9 @@ void DatabaseOrdinary::loadTables( using FileNames = std::vector; FileNames file_names; - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) - { - /// For '.svn', '.gitignore' directory and similar. - if (dir_it.name().at(0) == '.') - continue; - - /// There are .sql.bak files - skip them. - if (endsWith(dir_it.name(), ".sql.bak")) - continue; - - // There are files that we tried to delete previously - static const char * tmp_drop_ext = ".sql.tmp_drop"; - if (endsWith(dir_it.name(), tmp_drop_ext)) - { - const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext)); - if (Poco::File(getDataPath() + '/' + table_name).exists()) - { - Poco::File(dir_it->path()).renameTo(table_name + ".sql"); - LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously"); - } - else - { - LOG_INFO(log, "Removing file " << dir_it->path()); - Poco::File(dir_it->path()).remove(); - } - continue; - } - - /// There are files .sql.tmp - delete - if (endsWith(dir_it.name(), ".sql.tmp")) - { - LOG_INFO(log, "Removing file " << dir_it->path()); - Poco::File(dir_it->path()).remove(); - continue; - } - - /// The required files have names like `table_name.sql` - if (endsWith(dir_it.name(), ".sql")) - file_names.push_back(dir_it.name()); - else - throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + metadata_path, - ErrorCodes::INCORRECT_FILE_NAME); - } + DatabaseOnDisk::iterateTableFiles(*this, log, [&file_names](const String & file_name) { + file_names.push_back(file_name); + }); if (file_names.empty()) return; @@ -242,128 +195,17 @@ void DatabaseOrdinary::createTable( const StoragePtr & table, const ASTPtr & query) { - const auto & settings = context.getSettingsRef(); - - /// Create a file with metadata if necessary - if the query is not ATTACH. - /// Write the query of `ATTACH table` to it. - - /** The code is based on the assumption that all threads share the same order of operations - * - creating the .sql.tmp file; - * - adding a table to `tables`; - * - rename .sql.tmp to .sql. - */ - - /// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH. - /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery. - - if (isTableExist(context, table_name)) - throw Exception("Table " + getDatabaseName() + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - - String table_metadata_path = getTableMetadataPath(table_name); - String table_metadata_tmp_path = table_metadata_path + ".tmp"; - String statement; - - { - statement = getTableDefinitionFromCreateQuery(query); - - /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown. - WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); - writeString(statement, out); - out.next(); - if (settings.fsync_metadata) - out.sync(); - out.close(); - } - - try - { - /// Add a table to the map of known tables. - attachTable(table_name, table); - - /// If it was ATTACH query and file with table metadata already exist - /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one. - Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path); - } - catch (...) - { - Poco::File(table_metadata_tmp_path).remove(); - throw; - } + DatabaseOnDisk::createTable(*this, context, table_name, table, query); } void DatabaseOrdinary::removeTable( - const Context & /*context*/, + const Context & context, const String & table_name) { - StoragePtr res = detachTable(table_name); - - String table_metadata_path = getTableMetadataPath(table_name); - - try - { - Poco::File(table_metadata_path).remove(); - } - catch (...) - { - try - { - Poco::File(table_metadata_path + ".tmp_drop").remove(); - return; - } - catch (...) - { - LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); - } - attachTable(table_name, res); - throw; - } + DatabaseOnDisk::removeTable(*this, context, table_name, log); } -static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true) -{ - String query; - - try - { - ReadBufferFromFile in(metadata_path, 4096); - readStringUntilEOF(query, in); - } - catch (const Exception & e) - { - if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST) - return nullptr; - else - throw; - } - - ParserCreateQuery parser; - const char * pos = query.data(); - std::string error_message; - auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, - "in file " + metadata_path, /* allow_multi_statements = */ false, 0); - - if (!ast && throw_on_error) - throw Exception(error_message, ErrorCodes::SYNTAX_ERROR); - - return ast; -} - -static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error) -{ - ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error); - - if (ast) - { - auto & ast_create_query = ast->as(); - ast_create_query.attach = false; - ast_create_query.database = database; - } - - return ast; -} - - void DatabaseOrdinary::renameTable( const Context & context, const String & table_name, @@ -371,41 +213,7 @@ void DatabaseOrdinary::renameTable( const String & to_table_name, TableStructureWriteLockHolder & lock) { - DatabaseOrdinary * to_database_concrete = typeid_cast(&to_database); - - if (!to_database_concrete) - throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED); - - StoragePtr table = tryGetTable(context, table_name); - - if (!table) - throw Exception("Table " + getDatabaseName() + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - - /// Notify the table that it is renamed. If the table does not support renaming, exception is thrown. - try - { - table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/", - to_database_concrete->getDatabaseName(), - to_table_name, lock); - } - catch (const Exception &) - { - throw; - } - catch (const Poco::Exception & e) - { - /// Better diagnostics. - throw Exception{Exception::CreateFromPoco, e}; - } - - ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(getMetadataPath(), table_name)); - if (!ast) - throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST); - ast->as().table = to_table_name; - - /// NOTE Non-atomic. - to_database_concrete->createTable(context, to_table_name, table, ast); - removeTable(context, table_name); + DatabaseOnDisk::renameTable(*this, context, table_name, to_database, to_table_name, lock); } @@ -426,53 +234,19 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime( } } -ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context, - const String & table_name, bool throw_on_error) const -{ - ASTPtr ast; - - auto table_metadata_path = detail::getTableMetadataPath(getMetadataPath(), table_name); - ast = getCreateQueryFromMetadata(table_metadata_path, getDatabaseName(), throw_on_error); - if (!ast && throw_on_error) - { - /// Handle system.* tables for which there are no table.sql files. - bool has_table = tryGetTable(context, table_name) != nullptr; - - auto msg = has_table - ? "There is no CREATE TABLE query for table " - : "There is no metadata file for table "; - - throw Exception(msg + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY); - } - - return ast; -} - ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const { - return getCreateTableQueryImpl(context, table_name, true); + return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name); } ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const { - return getCreateTableQueryImpl(context, table_name, false); + return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name); } -ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const +ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & context) const { - ASTPtr ast; - - auto database_metadata_path = detail::getDatabaseMetadataPath(getMetadataPath()); - ast = getCreateQueryFromMetadata(database_metadata_path, getDatabaseName(), true); - if (!ast) - { - /// Handle databases (such as default) for which there are no database.sql files. - String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary"; - ParserCreateQuery parser; - ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0); - } - - return ast; + return DatabaseOnDisk::getCreateDatabaseQuery(*this, context); } void DatabaseOrdinary::alterTable( @@ -546,8 +320,7 @@ void DatabaseOrdinary::alterTable( void DatabaseOrdinary::drop() { - Poco::File(getDataPath()).remove(false); - Poco::File(getMetadataPath()).remove(false); + DatabaseOnDisk::drop(*this); }