thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

reduce copy-paste

Browse Source
This commit is contained in:
Vasilyev Nikita 2019-10-03 11:27:43 +03:00
parent 080815833f
commit 7ac9fadc0d
5 changed files with 71 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -74,8 +74,8 @@ void DatabaseLazy::loadTables(
Context & /* context */,
bool /* has_force_restore_data_flag */)
{
iterateTableFiles([this](const Poco::DirectoryIterator & dir_it) {
const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - 4);
DatabaseOnDisk::iterateTableFiles(*this, log, [this](const String & file_name) {
const std::string table_name = file_name.substr(0, file_name.size() - 4);
attachTable(table_name, nullptr);
});
}
@ -291,56 +291,6 @@ String DatabaseLazy::getTableMetadataPath(const String & table_name) const
return DatabaseOnDisk::getTableMetadataPath(*this, table_name);
}
void DatabaseLazy::iterateTableFiles(const IteratingFunction & iterating_function) const
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
{
/// For '.svn', '.gitignore' directory and similar.
if (dir_it.name().at(0) == '.')
continue;
/// There are .sql.bak files - skip them.
if (endsWith(dir_it.name(), ".sql.bak"))
continue;
// There are files that we tried to delete previously
static const char * tmp_drop_ext = ".sql.tmp_drop";
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(getDataPath() + '/' + table_name).exists())
{
Poco::File(dir_it->path()).renameTo(table_name + ".sql");
LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
}
else
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
}
continue;
}
/// There are files .sql.tmp - delete
if (endsWith(dir_it.name(), ".sql.tmp"))
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
continue;
}
/// The required files have names like `table_name.sql`
if (endsWith(dir_it.name(), ".sql"))
{
iterating_function(dir_it);
}
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + metadata_path,
ErrorCodes::INCORRECT_FILE_NAME);
}
}
StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table_name) const
{
clearExpiredTables();

3
dbms/src/Databases/DatabaseLazy.h
View File

@ -142,9 +142,6 @@ private:
Poco::Logger * log;
using IteratingFunction = std::function<void(const Poco::DirectoryIterator &)>;
void iterateTableFiles(const IteratingFunction & iterating_function) const;
StoragePtr loadTable(const Context & context, const String & table_name) const;
void clearExpiredTables() const;

57
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -22,11 +22,12 @@ namespace DB
namespace ErrorCodes
{
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int FILE_DOESNT_EXIST;
extern const int INCORRECT_FILE_NAME;
extern const int SYNTAX_ERROR;
extern const int TABLE_ALREADY_EXISTS;
extern const int UNKNOWN_TABLE;
extern const int FILE_DOESNT_EXIST;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int SYNTAX_ERROR;
}
@ -232,4 +233,54 @@ String DatabaseOnDisk::getTableMetadataPath(const IDatabase & database, const St
return detail::getTableMetadataPath(database.getMetadataPath(), table_name);
}
void DatabaseOnDisk::iterateTableFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function)
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it)
{
/// For '.svn', '.gitignore' directory and similar.
if (dir_it.name().at(0) == '.')
continue;
/// There are .sql.bak files - skip them.
if (endsWith(dir_it.name(), ".sql.bak"))
continue;
// There are files that we tried to delete previously
static const char * tmp_drop_ext = ".sql.tmp_drop";
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(database.getDataPath() + '/' + table_name).exists())
{
Poco::File(dir_it->path()).renameTo(table_name + ".sql");
LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
}
else
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
}
continue;
}
/// There are files .sql.tmp - delete
if (endsWith(dir_it.name(), ".sql.tmp"))
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
continue;
}
/// The required files have names like `table_name.sql`
if (endsWith(dir_it.name(), ".sql"))
{
iterating_function(dir_it.name());
}
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + database.getMetadataPath(),
ErrorCodes::INCORRECT_FILE_NAME);
}
}
}

3
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -64,6 +64,9 @@ public:
const IDatabase & database,
const String & table_name);
using IteratingFunction = std::function<void(const String &)>;
static void iterateTableFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function);
private:
static ASTPtr getCreateTableQueryImpl(
const IDatabase & database,

251
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -47,12 +47,6 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_TABLES = 256;
static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
namespace detail
{
extern String getTableMetadataPath(const String & base_path, const String & table_name);
extern String getDatabaseMetadataPath(const String & base_path);
}
static void loadTable(
Context & context,
const String & database_metadata_path,
@ -117,50 +111,9 @@ void DatabaseOrdinary::loadTables(
using FileNames = std::vector<std::string>;
FileNames file_names;
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
{
/// For '.svn', '.gitignore' directory and similar.
if (dir_it.name().at(0) == '.')
continue;
/// There are .sql.bak files - skip them.
if (endsWith(dir_it.name(), ".sql.bak"))
continue;
// There are files that we tried to delete previously
static const char * tmp_drop_ext = ".sql.tmp_drop";
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string table_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(getDataPath() + '/' + table_name).exists())
{
Poco::File(dir_it->path()).renameTo(table_name + ".sql");
LOG_WARNING(log, "Table " << backQuote(table_name) << " was not dropped previously");
}
else
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
}
continue;
}
/// There are files .sql.tmp - delete
if (endsWith(dir_it.name(), ".sql.tmp"))
{
LOG_INFO(log, "Removing file " << dir_it->path());
Poco::File(dir_it->path()).remove();
continue;
}
/// The required files have names like `table_name.sql`
if (endsWith(dir_it.name(), ".sql"))
file_names.push_back(dir_it.name());
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + metadata_path,
ErrorCodes::INCORRECT_FILE_NAME);
}
DatabaseOnDisk::iterateTableFiles(*this, log, [&file_names](const String & file_name) {
file_names.push_back(file_name);
});
if (file_names.empty())
return;
@ -242,128 +195,17 @@ void DatabaseOrdinary::createTable(
const StoragePtr & table,
const ASTPtr & query)
{
const auto & settings = context.getSettingsRef();
/// Create a file with metadata if necessary - if the query is not ATTACH.
/// Write the query of `ATTACH table` to it.
/** The code is based on the assumption that all threads share the same order of operations
* - creating the .sql.tmp file;
* - adding a table to `tables`;
* - rename .sql.tmp to .sql.
*/
/// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (isTableExist(context, table_name))
throw Exception("Table " + getDatabaseName() + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String table_metadata_path = getTableMetadataPath(table_name);
String table_metadata_tmp_path = table_metadata_path + ".tmp";
String statement;
{
statement = getTableDefinitionFromCreateQuery(query);
/// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (settings.fsync_metadata)
out.sync();
out.close();
}
try
{
/// Add a table to the map of known tables.
attachTable(table_name, table);
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
}
catch (...)
{
Poco::File(table_metadata_tmp_path).remove();
throw;
}
DatabaseOnDisk::createTable(*this, context, table_name, table, query);
}
void DatabaseOrdinary::removeTable(
const Context & /*context*/,
const Context & context,
const String & table_name)
{
StoragePtr res = detachTable(table_name);
String table_metadata_path = getTableMetadataPath(table_name);
try
{
Poco::File(table_metadata_path).remove();
}
catch (...)
{
try
{
Poco::File(table_metadata_path + ".tmp_drop").remove();
return;
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
}
attachTable(table_name, res);
throw;
}
DatabaseOnDisk::removeTable(*this, context, table_name, log);
}
static ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true)
{
String query;
try
{
ReadBufferFromFile in(metadata_path, 4096);
readStringUntilEOF(query, in);
}
catch (const Exception & e)
{
if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
return nullptr;
else
throw;
}
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + metadata_path, /* allow_multi_statements = */ false, 0);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
return ast;
}
static ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error)
{
ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error);
if (ast)
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
ast_create_query.database = database;
}
return ast;
}
void DatabaseOrdinary::renameTable(
const Context & context,
const String & table_name,
@ -371,41 +213,7 @@ void DatabaseOrdinary::renameTable(
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
DatabaseOrdinary * to_database_concrete = typeid_cast<DatabaseOrdinary *>(&to_database);
if (!to_database_concrete)
throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
StoragePtr table = tryGetTable(context, table_name);
if (!table)
throw Exception("Table " + getDatabaseName() + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/",
to_database_concrete->getDatabaseName(),
to_table_name, lock);
}
catch (const Exception &)
{
throw;
}
catch (const Poco::Exception & e)
{
/// Better diagnostics.
throw Exception{Exception::CreateFromPoco, e};
}
ASTPtr ast = getQueryFromMetadata(detail::getTableMetadataPath(getMetadataPath(), table_name));
if (!ast)
throw Exception("There is no metadata file for table " + table_name, ErrorCodes::FILE_DOESNT_EXIST);
ast->as<ASTCreateQuery &>().table = to_table_name;
/// NOTE Non-atomic.
to_database_concrete->createTable(context, to_table_name, table, ast);
removeTable(context, table_name);
DatabaseOnDisk::renameTable<DatabaseOrdinary>(*this, context, table_name, to_database, to_table_name, lock);
}
@ -426,53 +234,19 @@ time_t DatabaseOrdinary::getTableMetadataModificationTime(
}
}
ASTPtr DatabaseOrdinary::getCreateTableQueryImpl(const Context & context,
const String & table_name, bool throw_on_error) const
{
ASTPtr ast;
auto table_metadata_path = detail::getTableMetadataPath(getMetadataPath(), table_name);
ast = getCreateQueryFromMetadata(table_metadata_path, getDatabaseName(), throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_table = tryGetTable(context, table_name) != nullptr;
auto msg = has_table
? "There is no CREATE TABLE query for table "
: "There is no metadata file for table ";
throw Exception(msg + table_name, ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
}
return ast;
}
ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
{
return getCreateTableQueryImpl(context, table_name, true);
return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return getCreateTableQueryImpl(context, table_name, false);
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & /*context*/) const
ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & context) const
{
ASTPtr ast;
auto database_metadata_path = detail::getDatabaseMetadataPath(getMetadataPath());
ast = getCreateQueryFromMetadata(database_metadata_path, getDatabaseName(), true);
if (!ast)
{
/// Handle databases (such as default) for which there are no database.sql files.
String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
ParserCreateQuery parser;
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
}
return ast;
return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
}
void DatabaseOrdinary::alterTable(
@ -546,8 +320,7 @@ void DatabaseOrdinary::alterTable(
void DatabaseOrdinary::drop()
{
Poco::File(getDataPath()).remove(false);
Poco::File(getMetadataPath()).remove(false);
DatabaseOnDisk::drop(*this);
}