ClickHouse/src/Databases/DatabaseOnDisk.cpp

705 lines
26 KiB
C++
Raw Normal View History

2019-10-02 10:10:45 +00:00
#include <Databases/DatabaseOnDisk.h>
2019-10-03 09:31:59 +00:00
2019-10-02 10:10:45 +00:00
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
2019-10-10 17:33:01 +00:00
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
2019-10-02 10:10:45 +00:00
#include <Parsers/ASTCreateQuery.h>
2019-10-03 09:31:59 +00:00
#include <Parsers/ParserCreateQuery.h>
2019-10-10 17:33:01 +00:00
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
2019-10-02 10:10:45 +00:00
#include <Storages/IStorage.h>
2019-10-10 17:33:01 +00:00
#include <Storages/StorageFactory.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Common/escapeForFileName.h>
2019-10-02 10:10:45 +00:00
#include <common/logger_useful.h>
2019-11-26 19:06:19 +00:00
#include <Databases/DatabaseOrdinary.h>
#include <Databases/DatabaseAtomic.h>
#include <Common/assert_cast.h>
2021-04-30 17:20:53 +00:00
#include <filesystem>
2021-05-07 21:53:44 +00:00
#include <Poco/File.h>
#include <Poco/Path.h>
#include <Common/createFile.h>
2021-04-30 17:20:53 +00:00
namespace fs = std::filesystem;
2019-10-10 17:33:01 +00:00
2019-10-02 10:10:45 +00:00
namespace DB
{
2019-10-10 17:33:01 +00:00
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
2019-10-02 10:10:45 +00:00
namespace ErrorCodes
{
2020-02-25 18:02:41 +00:00
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int NOT_IMPLEMENTED;
extern const int LOGICAL_ERROR;
2019-10-03 08:27:43 +00:00
extern const int FILE_DOESNT_EXIST;
extern const int INCORRECT_FILE_NAME;
2019-10-02 10:10:45 +00:00
extern const int SYNTAX_ERROR;
2019-10-03 08:27:43 +00:00
extern const int TABLE_ALREADY_EXISTS;
2021-04-21 14:41:46 +00:00
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
2019-10-02 10:10:45 +00:00
}
2019-10-10 17:33:01 +00:00
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
2019-11-01 12:47:55 +00:00
const String & table_data_path_relative,
ContextPtr context,
2019-10-10 17:33:01 +00:00
bool has_force_restore_data_flag)
{
ast_create_query.attach = true;
ast_create_query.database = database_name;
if (ast_create_query.as_table_function)
{
const auto & factory = TableFunctionFactory::instance();
auto table_function = factory.get(ast_create_query.as_table_function, context);
ColumnsDescription columns;
if (ast_create_query.columns_list && ast_create_query.columns_list->columns)
columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true);
StoragePtr storage = table_function->execute(ast_create_query.as_table_function, context, ast_create_query.table, std::move(columns));
2020-04-07 14:05:51 +00:00
storage->renameInMemory(ast_create_query);
2019-10-10 17:33:01 +00:00
return {ast_create_query.table, storage};
}
2021-04-21 14:41:46 +00:00
ColumnsDescription columns;
ConstraintsDescription constraints;
if (!ast_create_query.is_dictionary)
{
/// We do not directly use `InterpreterCreateQuery::execute`, because
/// - the database has not been loaded yet;
/// - the code is simpler, since the query is already brought to a suitable form.
if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true);
constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
}
2019-10-10 17:33:01 +00:00
return
{
ast_create_query.table,
StorageFactory::instance().get(
ast_create_query,
2019-12-04 16:06:55 +00:00
table_data_path_relative,
context,
context->getGlobalContext(),
2019-12-04 16:06:55 +00:00
columns,
constraints,
has_force_restore_data_flag)
2019-10-10 17:33:01 +00:00
};
}
String getObjectDefinitionFromCreateQuery(const ASTPtr & query)
{
ASTPtr query_clone = query->clone();
2019-10-22 10:47:43 +00:00
auto * create = query_clone->as<ASTCreateQuery>();
2019-10-10 17:33:01 +00:00
2019-10-22 10:47:43 +00:00
if (!create)
{
2020-11-09 16:05:40 +00:00
WriteBufferFromOwnString query_buf;
formatAST(*query, query_buf, true);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Query '{}' is not CREATE query", query_buf.str());
2019-10-22 10:47:43 +00:00
}
if (!create->is_dictionary)
create->attach = true;
2019-10-11 15:22:30 +00:00
2019-10-10 17:33:01 +00:00
/// We remove everything that is not needed for ATTACH from the query.
2020-12-25 16:28:10 +00:00
assert(!create->temporary);
2019-10-22 10:47:43 +00:00
create->database.clear();
create->as_database.clear();
create->as_table.clear();
create->if_not_exists = false;
create->is_populate = false;
create->replace_view = false;
2020-12-25 16:28:10 +00:00
create->replace_table = false;
create->create_or_replace = false;
2019-10-10 17:33:01 +00:00
/// For views it is necessary to save the SELECT query itself, for the rest - on the contrary
2020-12-16 04:07:50 +00:00
if (!create->isView())
2019-10-22 10:47:43 +00:00
create->select = nullptr;
2019-10-10 17:33:01 +00:00
2019-10-22 10:47:43 +00:00
create->format = nullptr;
create->out_file = nullptr;
2019-10-10 17:33:01 +00:00
2020-01-17 16:07:20 +00:00
if (create->uuid != UUIDHelpers::Nil)
2019-12-02 19:11:18 +00:00
create->table = TABLE_WITH_UUID_NAME_PLACEHOLDER;
2020-11-09 16:05:40 +00:00
WriteBufferFromOwnString statement_buf;
formatAST(*create, statement_buf, false);
writeChar('\n', statement_buf);
return statement_buf.str();
2019-10-10 17:33:01 +00:00
}
2021-02-08 19:36:17 +00:00
void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemoryMetadata & metadata)
{
auto & ast_create_query = query->as<ASTCreateQuery &>();
bool has_structure = ast_create_query.columns_list && ast_create_query.columns_list->columns;
if (ast_create_query.as_table_function && !has_structure)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot alter table {} because it was created AS table function"
" and doesn't have structure in metadata", backQuote(ast_create_query.table));
assert(has_structure);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.secondary_indices);
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(metadata.constraints);
ast_create_query.columns_list->replace(ast_create_query.columns_list->columns, new_columns);
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices);
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints);
if (metadata.select.select_query)
{
query->replace(ast_create_query.select, metadata.select.select_query);
}
/// MaterializedView is one type of CREATE query without storage.
if (ast_create_query.storage)
{
ASTStorage & storage_ast = *ast_create_query.storage;
bool is_extended_storage_def
= storage_ast.partition_by || storage_ast.primary_key || storage_ast.order_by || storage_ast.sample_by || storage_ast.settings;
if (is_extended_storage_def)
{
if (metadata.sorting_key.definition_ast)
storage_ast.set(storage_ast.order_by, metadata.sorting_key.definition_ast);
if (metadata.primary_key.definition_ast)
storage_ast.set(storage_ast.primary_key, metadata.primary_key.definition_ast);
if (metadata.sampling_key.definition_ast)
storage_ast.set(storage_ast.sample_by, metadata.sampling_key.definition_ast);
if (metadata.table_ttl.definition_ast)
storage_ast.set(storage_ast.ttl_table, metadata.table_ttl.definition_ast);
else if (storage_ast.ttl_table != nullptr) /// TTL was removed
storage_ast.ttl_table = nullptr;
if (metadata.settings_changes)
storage_ast.set(storage_ast.settings, metadata.settings_changes);
}
}
}
DatabaseOnDisk::DatabaseOnDisk(
const String & name,
const String & metadata_path_,
const String & data_path_,
const String & logger,
ContextPtr local_context)
: DatabaseWithOwnTablesBase(name, logger, local_context)
2019-11-28 19:40:51 +00:00
, metadata_path(metadata_path_)
2020-04-06 23:22:44 +00:00
, data_path(data_path_)
2019-11-28 19:40:51 +00:00
{
2021-04-30 17:20:53 +00:00
fs::create_directories(local_context->getPath() + data_path);
fs::create_directories(metadata_path);
2019-11-28 19:40:51 +00:00
}
2019-10-03 07:59:48 +00:00
void DatabaseOnDisk::createTable(
ContextPtr local_context,
2019-10-03 07:59:48 +00:00
const String & table_name,
const StoragePtr & table,
const ASTPtr & query)
{
const auto & settings = local_context->getSettingsRef();
2020-03-23 00:12:13 +00:00
const auto & create = query->as<ASTCreateQuery &>();
2020-07-07 12:11:58 +00:00
assert(table_name == create.table);
2019-10-03 07:59:48 +00:00
/// Create a file with metadata if necessary - if the query is not ATTACH.
/// Write the query of `ATTACH table` to it.
/** The code is based on the assumption that all threads share the same order of operations
* - creating the .sql.tmp file;
* - adding a table to `tables`;
* - rename .sql.tmp to .sql.
*/
/// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (isTableExist(table_name, getContext()))
throw Exception(
ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists", backQuote(getDatabaseName()), backQuote(table_name));
2020-11-30 17:52:32 +00:00
2020-12-15 17:06:23 +00:00
String table_metadata_path = getObjectMetadataPath(table_name);
2019-10-03 07:59:48 +00:00
2020-03-23 00:12:13 +00:00
if (create.attach_short_syntax)
{
/// Metadata already exists, table was detached
removeDetachedPermanentlyFlag(local_context, table_name, table_metadata_path, true);
2020-03-23 00:12:13 +00:00
attachTable(table_name, table, getTableDataPath(create));
return;
}
2020-12-15 17:06:23 +00:00
if (!create.attach)
checkMetadataFilenameAvailability(table_name);
2021-04-30 17:20:53 +00:00
if (create.attach && fs::exists(table_metadata_path))
2020-12-15 17:06:23 +00:00
{
ASTPtr ast_detached = parseQueryFromMetadata(log, local_context, table_metadata_path);
2020-12-15 17:06:23 +00:00
auto & create_detached = ast_detached->as<ASTCreateQuery &>();
// either both should be Nil, either values should be equal
if (create.uuid != create_detached.uuid)
throw Exception(
ErrorCodes::TABLE_ALREADY_EXISTS,
"Table {}.{} already exist (detached permanently). To attach it back "
"you need to use short ATTACH syntax or a full statement with the same UUID",
backQuote(getDatabaseName()), backQuote(table_name));
}
2020-01-22 11:30:11 +00:00
String table_metadata_tmp_path = table_metadata_path + create_suffix;
2019-10-03 07:59:48 +00:00
String statement;
{
2019-10-10 17:33:01 +00:00
statement = getObjectDefinitionFromCreateQuery(query);
2019-10-03 07:59:48 +00:00
/// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (settings.fsync_metadata)
out.sync();
out.close();
}
commitCreateTable(create, table, table_metadata_tmp_path, table_metadata_path, local_context);
2020-12-15 17:06:23 +00:00
removeDetachedPermanentlyFlag(local_context, table_name, table_metadata_path, false);
2020-12-15 17:06:23 +00:00
}
/// If the table was detached permanently we will have a flag file with
/// .sql.detached extension, is not needed anymore since we attached the table back
void DatabaseOnDisk::removeDetachedPermanentlyFlag(ContextPtr, const String & table_name, const String & table_metadata_path, bool) const
2020-12-15 17:06:23 +00:00
{
try
{
2021-04-30 17:20:53 +00:00
fs::path detached_permanently_flag(table_metadata_path + detached_suffix);
2020-12-15 17:06:23 +00:00
2021-04-30 17:20:53 +00:00
if (fs::exists(detached_permanently_flag))
fs::remove(detached_permanently_flag);
2020-12-15 17:06:23 +00:00
}
catch (Exception & e)
{
e.addMessage("while trying to remove permanently detached flag. Table {}.{} may still be marked as permanently detached, and will not be reattached during server restart.", backQuote(getDatabaseName()), backQuote(table_name));
2020-12-15 17:06:23 +00:00
throw;
}
2020-03-23 00:12:13 +00:00
}
void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table,
2020-11-20 16:06:27 +00:00
const String & table_metadata_tmp_path, const String & table_metadata_path,
ContextPtr /*query_context*/)
2020-03-23 00:12:13 +00:00
{
2019-10-03 07:59:48 +00:00
try
{
/// Add a table to the map of known tables.
2020-03-23 00:12:13 +00:00
attachTable(query.table, table, getTableDataPath(query));
2019-10-03 07:59:48 +00:00
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
2021-04-30 17:20:53 +00:00
fs::rename(table_metadata_tmp_path, table_metadata_path);
2019-10-03 07:59:48 +00:00
}
catch (...)
{
2021-04-30 17:20:53 +00:00
fs::remove(table_metadata_tmp_path);
2019-10-03 07:59:48 +00:00
throw;
}
}
void DatabaseOnDisk::detachTablePermanently(ContextPtr, const String & table_name)
2020-11-30 17:52:32 +00:00
{
2020-12-14 14:21:31 +00:00
auto table = detachTable(table_name);
2020-11-30 17:52:32 +00:00
2021-05-07 21:53:44 +00:00
fs::path detached_permanently_flag(getObjectMetadataPath(table_name) + detached_suffix);
2020-11-30 17:52:32 +00:00
try
{
2021-05-07 21:53:44 +00:00
fs::createFile(detached_permanently_flag);
2020-11-30 17:52:32 +00:00
}
catch (Exception & e)
{
e.addMessage("while trying to set permanently detached flag. Table {}.{} may be reattached during server restart.", backQuote(getDatabaseName()), backQuote(table_name));
2020-11-30 17:52:32 +00:00
throw;
}
2019-10-03 07:59:48 +00:00
}
void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_name, bool /*no_delay*/)
2019-10-03 07:59:48 +00:00
{
String table_metadata_path = getObjectMetadataPath(table_name);
2020-01-22 11:30:11 +00:00
String table_metadata_path_drop = table_metadata_path + drop_suffix;
String table_data_path_relative = getTableDataPath(table_name);
2020-07-16 21:41:26 +00:00
if (table_data_path_relative.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Path is empty");
2019-10-03 07:59:48 +00:00
2020-01-22 11:30:11 +00:00
StoragePtr table = detachTable(table_name);
2020-09-03 01:06:12 +00:00
/// This is possible for Lazy database.
if (!table)
return;
2020-01-22 11:30:11 +00:00
bool renamed = false;
2019-10-03 07:59:48 +00:00
try
{
2021-04-30 17:20:53 +00:00
fs::rename(table_metadata_path, table_metadata_path_drop);
2020-01-22 11:30:11 +00:00
renamed = true;
table->drop();
table->is_dropped = true;
2021-04-30 17:20:53 +00:00
fs::path table_data_dir(local_context->getPath() + table_data_path_relative);
if (fs::exists(table_data_dir))
fs::remove_all(table_data_dir);
2019-10-03 07:59:48 +00:00
}
catch (...)
{
2020-05-23 22:24:01 +00:00
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
2020-01-22 11:30:11 +00:00
attachTable(table_name, table, table_data_path_relative);
if (renamed)
2021-04-30 17:20:53 +00:00
fs::rename(table_metadata_path_drop, table_metadata_path);
2019-10-03 07:59:48 +00:00
throw;
}
2020-01-22 11:30:11 +00:00
2021-04-30 17:20:53 +00:00
fs::remove(table_metadata_path_drop);
2019-10-03 07:59:48 +00:00
}
void DatabaseOnDisk::checkMetadataFilenameAvailability(const String & to_table_name) const
{
std::unique_lock lock(mutex);
2020-12-15 17:06:23 +00:00
checkMetadataFilenameAvailabilityUnlocked(to_table_name, lock);
}
void DatabaseOnDisk::checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name, std::unique_lock<std::mutex> &) const
{
2020-12-14 14:21:31 +00:00
String table_metadata_path = getObjectMetadataPath(to_table_name);
2021-04-30 17:20:53 +00:00
if (fs::exists(table_metadata_path))
{
2021-04-30 17:20:53 +00:00
fs::path detached_permanently_flag(table_metadata_path + detached_suffix);
2021-04-30 17:20:53 +00:00
if (fs::exists(detached_permanently_flag))
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached permanently)", backQuote(database_name), backQuote(to_table_name));
else
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists (detached)", backQuote(database_name), backQuote(to_table_name));
}
}
void DatabaseOnDisk::renameTable(
ContextPtr local_context,
const String & table_name,
IDatabase & to_database,
2020-03-31 20:38:05 +00:00
const String & to_table_name,
2020-07-16 21:41:26 +00:00
bool exchange,
bool dictionary)
2019-10-10 17:33:01 +00:00
{
2020-03-31 20:38:05 +00:00
if (exchange)
throw Exception("Tables can be exchanged only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
2020-07-16 21:41:26 +00:00
if (dictionary)
throw Exception("Dictionaries can be renamed only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
2020-03-31 20:38:05 +00:00
2019-11-26 19:06:19 +00:00
bool from_ordinary_to_atomic = false;
2019-12-02 19:11:18 +00:00
bool from_atomic_to_ordinary = false;
if (typeid(*this) != typeid(to_database))
2019-11-26 19:06:19 +00:00
{
if (typeid_cast<DatabaseOrdinary *>(this) && typeid_cast<DatabaseAtomic *>(&to_database))
from_ordinary_to_atomic = true;
2019-12-02 19:11:18 +00:00
else if (typeid_cast<DatabaseAtomic *>(this) && typeid_cast<DatabaseOrdinary *>(&to_database))
from_atomic_to_ordinary = true;
2021-02-11 22:23:40 +00:00
else if (dynamic_cast<DatabaseAtomic *>(this) && typeid_cast<DatabaseOrdinary *>(&to_database) && getEngineName() == "Replicated")
from_atomic_to_ordinary = true;
2019-11-26 19:06:19 +00:00
else
throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
2020-01-22 11:30:11 +00:00
auto table_data_relative_path = getTableDataPath(table_name);
2020-06-18 16:10:47 +00:00
TableExclusiveLockHolder table_lock;
2020-01-22 11:30:11 +00:00
String table_metadata_path;
ASTPtr attach_query;
2020-03-18 17:38:52 +00:00
/// DatabaseLazy::detachTable may return nullptr even if table exists, so we need tryGetTable for this case.
StoragePtr table = tryGetTable(table_name, getContext());
2020-03-18 17:38:52 +00:00
detachTable(table_name);
2021-02-11 22:23:40 +00:00
UUID prev_uuid = UUIDHelpers::Nil;
2019-10-10 17:33:01 +00:00
try
{
table_lock = table->lockExclusively(
local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout);
2020-01-22 11:30:11 +00:00
table_metadata_path = getObjectMetadataPath(table_name);
attach_query = parseQueryFromMetadata(log, local_context, table_metadata_path);
2020-01-22 11:30:11 +00:00
auto & create = attach_query->as<ASTCreateQuery &>();
2020-03-23 22:40:40 +00:00
create.database = to_database.getDatabaseName();
2020-01-22 11:30:11 +00:00
create.table = to_table_name;
if (from_ordinary_to_atomic)
create.uuid = UUIDHelpers::generateV4();
if (from_atomic_to_ordinary)
2021-02-11 22:23:40 +00:00
std::swap(create.uuid, prev_uuid);
2020-01-22 11:30:11 +00:00
if (auto * target_db = dynamic_cast<DatabaseOnDisk *>(&to_database))
target_db->checkMetadataFilenameAvailability(to_table_name);
2020-01-22 11:30:11 +00:00
/// Notify the table that it is renamed. It will move data to new path (if it stores data on disk) and update StorageID
2020-04-07 14:05:51 +00:00
table->rename(to_database.getTableDataPath(create), StorageID(create));
2019-10-10 17:33:01 +00:00
}
catch (const Exception &)
2019-10-10 17:33:01 +00:00
{
2020-01-22 11:30:11 +00:00
attachTable(table_name, table, table_data_relative_path);
2019-10-10 17:33:01 +00:00
throw;
}
catch (const Poco::Exception & e)
{
2020-01-22 11:30:11 +00:00
attachTable(table_name, table, table_data_relative_path);
/// Better diagnostics.
throw Exception{Exception::CreateFromPocoTag{}, e};
}
2019-10-10 17:33:01 +00:00
2020-01-22 11:30:11 +00:00
/// Now table data are moved to new database, so we must add metadata and attach table to new database
to_database.createTable(local_context, to_table_name, table, attach_query);
2020-01-22 11:30:11 +00:00
2021-04-30 17:20:53 +00:00
fs::remove(table_metadata_path);
2021-02-11 22:23:40 +00:00
if (from_atomic_to_ordinary)
{
2021-02-16 14:05:58 +00:00
auto & atomic_db = dynamic_cast<DatabaseAtomic &>(*this);
2021-02-11 22:23:40 +00:00
/// Special case: usually no actions with symlinks are required when detaching/attaching table,
/// but not when moving from Atomic database to Ordinary
if (table->storesDataOnDisk())
atomic_db.tryRemoveSymlink(table_name);
/// Forget about UUID, now it's possible to reuse it for new table
DatabaseCatalog::instance().removeUUIDMappingFinally(prev_uuid);
atomic_db.setDetachedTableNotInUseForce(prev_uuid);
}
}
2020-11-30 17:52:32 +00:00
/// It returns create table statement (even if table is detached)
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const String & table_name, ContextPtr, bool throw_on_error) const
2019-10-03 07:59:48 +00:00
{
ASTPtr ast;
bool has_table = tryGetTable(table_name, getContext()) != nullptr;
auto table_metadata_path = getObjectMetadataPath(table_name);
try
2019-10-03 07:59:48 +00:00
{
ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error);
}
catch (const Exception & e)
{
if (!has_table && e.code() == ErrorCodes::FILE_DOESNT_EXIST && throw_on_error)
throw Exception{"Table " + backQuote(table_name) + " doesn't exist",
ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY};
else if (throw_on_error)
throw;
2019-10-03 07:59:48 +00:00
}
return ast;
}
ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
2019-10-03 07:59:48 +00:00
{
ASTPtr ast;
auto settings = getContext()->getSettingsRef();
2020-07-07 12:11:58 +00:00
{
std::lock_guard lock(mutex);
auto database_metadata_path = getContext()->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql";
ast = parseQueryFromMetadata(log, getContext(), database_metadata_path, true);
2020-07-07 12:11:58 +00:00
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
ast_create_query.database = database_name;
}
2019-10-03 07:59:48 +00:00
if (!ast)
{
/// Handle databases (such as default) for which there are no database.sql files.
2019-12-02 19:11:18 +00:00
/// If database.sql doesn't exist, then engine is Ordinary
String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
2019-10-03 07:59:48 +00:00
ParserCreateQuery parser;
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth);
2019-10-03 07:59:48 +00:00
}
return ast;
}
void DatabaseOnDisk::drop(ContextPtr local_context)
2019-10-03 07:59:48 +00:00
{
2020-08-18 15:15:27 +00:00
assert(tables.empty());
2021-04-30 17:20:53 +00:00
fs::remove(local_context->getPath() + getDataPath());
fs::remove(getMetadataPath());
2019-10-03 07:59:48 +00:00
}
2020-03-08 22:38:12 +00:00
String DatabaseOnDisk::getObjectMetadataPath(const String & object_name) const
2019-10-03 07:59:48 +00:00
{
2020-03-08 22:38:12 +00:00
return getMetadataPath() + escapeForFileName(object_name) + ".sql";
2019-10-02 10:10:45 +00:00
}
2020-03-08 22:38:12 +00:00
time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_name) const
2019-10-03 08:35:58 +00:00
{
2020-03-08 22:38:12 +00:00
String table_metadata_path = getObjectMetadataPath(object_name);
2019-10-03 08:35:58 +00:00
Poco::File meta_file(table_metadata_path);
if (meta_file.exists())
return meta_file.getLastModified().epochTime();
else
return static_cast<time_t>(0);
}
void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const IteratingFunction & process_metadata_file) const
2020-01-23 19:10:09 +00:00
{
2020-04-06 23:22:44 +00:00
auto process_tmp_drop_metadata_file = [&](const String & file_name)
2020-01-23 19:10:09 +00:00
{
assert(getUUID() == UUIDHelpers::Nil);
2020-01-23 19:10:09 +00:00
static const char * tmp_drop_ext = ".sql.tmp_drop";
const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext));
2021-04-30 17:20:53 +00:00
if (fs::exists(local_context->getPath() + getDataPath() + '/' + object_name))
2020-01-23 19:10:09 +00:00
{
2021-04-30 17:20:53 +00:00
fs::rename(getMetadataPath() + file_name, getMetadataPath() + object_name + ".sql");
2020-05-23 22:24:01 +00:00
LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name));
2020-01-23 19:10:09 +00:00
process_metadata_file(object_name + ".sql");
}
else
{
2020-05-23 22:24:01 +00:00
LOG_INFO(log, "Removing file {}", getMetadataPath() + file_name);
2021-04-30 17:20:53 +00:00
fs::remove(getMetadataPath() + file_name);
2020-01-23 19:10:09 +00:00
}
};
/// Metadata files to load: name and flag for .tmp_drop files
std::set<std::pair<String, bool>> metadata_files;
2021-05-07 21:53:44 +00:00
fs::directory_iterator dir_end;
for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
2019-10-03 08:27:43 +00:00
{
2021-05-07 21:53:44 +00:00
String file_name = dir_it->path().filename();
2019-10-03 08:27:43 +00:00
/// For '.svn', '.gitignore' directory and similar.
2021-05-07 21:53:44 +00:00
if (file_name.at(0) == '.')
2019-10-03 08:27:43 +00:00
continue;
/// There are .sql.bak files - skip them.
2021-05-07 21:53:44 +00:00
if (endsWith(file_name, ".sql.bak"))
2019-10-03 08:27:43 +00:00
continue;
/// Permanently detached table flag
2021-05-07 21:53:44 +00:00
if (endsWith(file_name, ".sql.detached"))
2020-11-30 17:52:32 +00:00
continue;
2021-05-07 21:53:44 +00:00
if (endsWith(file_name, ".sql.tmp_drop"))
2019-10-03 08:27:43 +00:00
{
2020-01-23 19:10:09 +00:00
/// There are files that we tried to delete previously
2021-05-07 21:53:44 +00:00
metadata_files.emplace(file_name, false);
2019-10-03 08:27:43 +00:00
}
2021-05-07 21:53:44 +00:00
else if (endsWith(file_name, ".sql.tmp"))
2019-10-03 08:27:43 +00:00
{
2020-01-23 19:10:09 +00:00
/// There are files .sql.tmp - delete
2021-05-07 21:53:44 +00:00
LOG_INFO(log, "Removing file {}", dir_it->path().string());
2021-04-30 17:20:53 +00:00
fs::remove(dir_it->path());
2019-10-03 08:27:43 +00:00
}
2021-05-07 21:53:44 +00:00
else if (endsWith(file_name, ".sql"))
2019-10-03 08:27:43 +00:00
{
2020-01-23 19:10:09 +00:00
/// The required files have names like `table_name.sql`
2021-05-07 21:53:44 +00:00
metadata_files.emplace(file_name, true);
2019-10-03 08:27:43 +00:00
}
else
2021-05-07 21:53:44 +00:00
throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath());
2019-10-03 08:27:43 +00:00
}
/// Read and parse metadata in parallel
ThreadPool pool;
for (const auto & file : metadata_files)
{
pool.scheduleOrThrowOnError([&]()
{
if (file.second)
process_metadata_file(file.first);
else
process_tmp_drop_metadata_file(file.first);
});
}
pool.wait();
2019-10-03 08:27:43 +00:00
}
ASTPtr DatabaseOnDisk::parseQueryFromMetadata(
Poco::Logger * logger,
ContextPtr local_context,
const String & metadata_file_path,
bool throw_on_error /*= true*/,
bool remove_empty /*= false*/)
{
String query;
try
{
2019-12-02 19:11:18 +00:00
ReadBufferFromFile in(metadata_file_path, METADATA_FILE_BUFFER_SIZE);
readStringUntilEOF(query, in);
}
catch (const Exception & e)
{
if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
return nullptr;
else
throw;
}
2019-12-02 19:11:18 +00:00
/** Empty files with metadata are generated after a rough restart of the server.
* Remove these files to slightly reduce the work of the admins on startup.
*/
if (remove_empty && query.empty())
{
if (logger)
LOG_ERROR(logger, "File {} is empty. Removing.", metadata_file_path);
2021-04-30 17:20:53 +00:00
fs::remove(metadata_file_path);
2019-12-02 19:11:18 +00:00
return nullptr;
}
auto settings = local_context->getSettingsRef();
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
2020-03-19 21:14:52 +00:00
"in file " + metadata_file_path, /* allow_multi_statements = */ false, 0, settings.max_parser_depth);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
2019-12-02 19:11:18 +00:00
else if (!ast)
return nullptr;
auto & create = ast->as<ASTCreateQuery &>();
2020-07-02 20:39:31 +00:00
if (!create.table.empty() && create.uuid != UUIDHelpers::Nil)
2019-12-02 19:11:18 +00:00
{
String table_name = Poco::Path(metadata_file_path).makeFile().getBaseName();
2020-03-18 17:38:52 +00:00
table_name = unescapeForFileName(table_name);
2020-01-23 19:10:09 +00:00
if (create.table != TABLE_WITH_UUID_NAME_PLACEHOLDER && logger)
LOG_WARNING(
logger,
"File {} contains both UUID and table name. Will use name `{}` instead of `{}`",
metadata_file_path,
table_name,
create.table);
2019-12-02 19:11:18 +00:00
create.table = table_name;
}
return ast;
}
ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const
{
ASTPtr ast = parseQueryFromMetadata(log, getContext(), database_metadata_path, throw_on_error);
if (ast)
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
2020-07-07 12:11:58 +00:00
ast_create_query.database = getDatabaseName();
}
return ast;
}
2019-10-05 10:21:47 +00:00
}