thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-18 13:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

retry DROP after restart

Browse Source
This commit is contained in:
Alexander Tokmakov 2020-01-23 22:10:09 +03:00
parent 34e733f06d
commit e4a889c7f4
5 changed files with 68 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
dbms/src/Databases/DatabaseAtomic.cpp
View File

@ -3,6 +3,7 @@
#include <Poco/File.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Common/Stopwatch.h>
namespace DB
@ -80,7 +81,7 @@ void DatabaseAtomic::dropTable(const Context & context, const String & table_nam
// 1. CREATE table_name: + table_name.sql
// 2. DROP table_name: table_name.sql -> table_name.sql.tmp_drop
// 3. CREATE table_name: + table_name.sql
// 4. DROP table_name: table_name.sql -> table_name.sql.tmp_drop overwrites table_name.sql.tmp_drop ?
// 4. DROP table_name: table_name.sql -> table_name.sql.tmp_drop overwrites table_name.sql.tmp_drop
Poco::File(table_metadata_path).renameTo(table_metadata_path_drop);
{
@ -127,6 +128,27 @@ void DatabaseAtomic::renameTable(const Context & context, const String & table_n
void DatabaseAtomic::loadStoredObjects(Context & context, bool has_force_restore_data_flag)
{
iterateMetadataFiles(context, [](const String &) {}, [&](const String & file_name)
{
String full_path = getMetadataPath() + file_name;
LOG_INFO(log, "Trying load partially dropped table from " << full_path);
try
{
auto ast = parseQueryFromMetadata(context, full_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (!ast) //TODO why?
return;
auto & query = ast->as<ASTCreateQuery &>();
auto [_, table] = createTableFromAST(query, database_name, getTableDataPath(query), context, has_force_restore_data_flag);
time_t drop_time = Poco::File(full_path).getLastModified().epochTime();
tables_to_drop.push_back({table, context.getPath() + getTableDataPath(query), drop_time});
}
catch (Exception & e)
{
e.addMessage("Cannot complete table drop " + full_path);
//TODO may be remove data dir (if UUID successfully parsed) and .tmp_drop metadata?
}
});
DatabaseOrdinary::loadStoredObjects(context, has_force_restore_data_flag);
drop_task->activateAndSchedule();
}

63
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -342,7 +342,33 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_na
return static_cast<time_t>(0);
}
void DatabaseOnDisk::iterateMetadataFiles(const Context & /*context*/, const IteratingFunction & iterating_function) const
void DatabaseOnDisk::iterateMetadataFiles(const Context & context,
const DatabaseOnDisk::IteratingFunction & process_metadata_file) const
{
IteratingFunction process_tmp_drop_metadata_file = [&](const String & file_name)
{
static const char * tmp_drop_ext = ".sql.tmp_drop";
const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext));
if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
{
Poco::File(getMetadataPath() + file_name).renameTo(context.getPath() + getMetadataPath() + object_name + ".sql");
LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
process_metadata_file(object_name + ".sql");
}
else
{
LOG_INFO(log, "Removing file " << getMetadataPath() + file_name);
Poco::File(getMetadataPath() + file_name).remove();
}
};
IteratingFunction do_nothing = [](const String &){};
//FIXME refactor this trash
iterateMetadataFiles(context, process_metadata_file, dynamic_cast<const DatabaseAtomic *>(this) ? do_nothing : process_tmp_drop_metadata_file);
}
void DatabaseOnDisk::iterateMetadataFiles(const Context & /*context*/, const IteratingFunction & process_metadata_file,
const IteratingFunction & process_tmp_drop_metadata_file) const
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
@ -355,39 +381,22 @@ void DatabaseOnDisk::iterateMetadataFiles(const Context & /*context*/, const Ite
if (endsWith(dir_it.name(), ".sql.bak"))
continue;
// There are files that we tried to delete previously
static const char * tmp_drop_ext = ".sql.tmp_drop";
if (endsWith(dir_it.name(), tmp_drop_ext))
{
//const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
//if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
//{
// /// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
// //TODO check all paths
// Poco::File(dir_it->path()).renameTo(context.getPath() + getMetadataPath() + object_name + ".sql");
// LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
// iterating_function(object_name + ".sql");
//}
//else
//{
// LOG_INFO(log, "Removing file " << dir_it->path());
// Poco::File(dir_it->path()).remove();
//}
continue;
/// There are files that we tried to delete previously
process_tmp_drop_metadata_file(dir_it.name());
}
/// There are files .sql.tmp - delete
if (endsWith(dir_it.name(), ".sql.tmp"))
else if (endsWith(dir_it.name(), ".sql.tmp"))
{
/// There are files .sql.tmp - delete
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
continue;
}
/// The required files have names like `table_name.sql`
if (endsWith(dir_it.name(), ".sql"))
else if (endsWith(dir_it.name(), ".sql"))
{
iterating_function(dir_it.name());
/// The required files have names like `table_name.sql`
process_metadata_file(dir_it.name());
}
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
@ -437,7 +446,11 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const Context & context, const Str
auto & create = ast->as<ASTCreateQuery &>();
if (create.uuid != UUIDHelpers::Nil)
{
//FIXME it can be .sql or .sql.tmp_drop
String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName();
if (Poco::Path(table_name).makeFile().getExtension() == "sql")
table_name = Poco::Path(table_name).makeFile().getBaseName();
if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER)
LOG_WARNING(log, "File " << metadata_file_path << " contains both UUID and table name. "
"Will use name `" << table_name << "` instead of `" << create.table << "`");

6
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -66,7 +66,10 @@ protected:
static constexpr const char * drop_suffix = ".tmp_drop";
using IteratingFunction = std::function<void(const String &)>;
void iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const;
void iterateMetadataFiles(const Context & context, const IteratingFunction & process_metadata_file) const;
void iterateMetadataFiles(const Context & context, const IteratingFunction & process_metadata_file,
const IteratingFunction & process_tmp_drop_metadata_file) const;
ASTPtr getCreateTableQueryImpl(
const Context & context,
@ -76,6 +79,7 @@ protected:
ASTPtr parseQueryFromMetadata(const Context & context, const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const;
ASTPtr getCreateQueryFromMetadata(const Context & context, const String & metadata_path, bool throw_on_error) const;
//bool detachTableAndRemoveMetadata(const String & table_name);
//void replaceMetadata(const ASTPtr & create, );

1
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -29,6 +29,7 @@
#include <Common/typeid_cast.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include "DatabaseAtomic.h"
namespace DB

2
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -27,7 +27,7 @@ public:
const String & name,
const StorageInMemoryMetadata & metadata) override;
private:
protected:
void startupTables(ThreadPool & thread_pool);
};