2018-06-05 19:46:49 +00:00
|
|
|
#include <iomanip>
|
|
|
|
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Core/Settings.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Databases/DatabaseMemory.h>
|
2019-10-02 12:58:13 +00:00
|
|
|
#include <Databases/DatabaseOnDisk.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Databases/DatabaseOrdinary.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Databases/DatabasesCommon.h>
|
|
|
|
#include <IO/ReadBufferFromFile.h>
|
2018-06-05 19:46:49 +00:00
|
|
|
#include <IO/ReadHelpers.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <IO/WriteBufferFromFile.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Interpreters/InterpreterCreateQuery.h>
|
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/ParserCreateQuery.h>
|
|
|
|
#include <Parsers/parseQuery.h>
|
|
|
|
#include <Storages/IStorage.h>
|
|
|
|
|
|
|
|
#include <Poco/DirectoryIterator.h>
|
|
|
|
#include <Poco/Event.h>
|
|
|
|
#include <Common/Stopwatch.h>
|
|
|
|
#include <Common/StringUtils/StringUtils.h>
|
|
|
|
#include <Common/ThreadPool.h>
|
|
|
|
#include <Common/escapeForFileName.h>
|
|
|
|
#include <Common/typeid_cast.h>
|
|
|
|
#include <common/logger_useful.h>
|
2018-12-05 11:20:55 +00:00
|
|
|
#include <ext/scope_guard.h>
|
2016-03-19 01:18:49 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int TABLE_ALREADY_EXISTS;
|
|
|
|
extern const int UNKNOWN_TABLE;
|
|
|
|
extern const int CANNOT_CREATE_TABLE_FROM_METADATA;
|
|
|
|
extern const int INCORRECT_FILE_NAME;
|
2018-01-30 17:47:04 +00:00
|
|
|
extern const int FILE_DOESNT_EXIST;
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int LOGICAL_ERROR;
|
2018-01-30 17:47:04 +00:00
|
|
|
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
|
2018-03-23 19:56:24 +00:00
|
|
|
extern const int SYNTAX_ERROR;
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static constexpr size_t PRINT_MESSAGE_EACH_N_TABLES = 256;
|
|
|
|
static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
|
|
|
|
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
|
|
|
|
|
|
|
|
static void loadTable(
|
2017-04-01 07:20:54 +00:00
|
|
|
Context & context,
|
|
|
|
const String & database_metadata_path,
|
|
|
|
DatabaseOrdinary & database,
|
|
|
|
const String & database_name,
|
|
|
|
const String & database_data_path,
|
|
|
|
const String & file_name,
|
|
|
|
bool has_force_restore_data_flag)
|
2016-03-19 01:18:49 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
Logger * log = &Logger::get("loadTable");
|
|
|
|
|
|
|
|
const String table_metadata_path = database_metadata_path + "/" + file_name;
|
|
|
|
|
|
|
|
String s;
|
|
|
|
{
|
|
|
|
char in_buf[METADATA_FILE_BUFFER_SIZE];
|
|
|
|
ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
|
|
|
|
readStringUntilEOF(s, in);
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Empty files with metadata are generated after a rough restart of the server.
|
|
|
|
* Remove these files to slightly reduce the work of the admins on startup.
|
|
|
|
*/
|
|
|
|
if (s.empty())
|
|
|
|
{
|
|
|
|
LOG_ERROR(log, "File " << table_metadata_path << " is empty. Removing.");
|
|
|
|
Poco::File(table_metadata_path).remove();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
String table_name;
|
|
|
|
StoragePtr table;
|
|
|
|
std::tie(table_name, table) = createTableFromDefinition(
|
|
|
|
s, database_name, database_data_path, context, has_force_restore_data_flag, "in file " + table_metadata_path);
|
|
|
|
database.attachTable(table_name, table);
|
|
|
|
}
|
|
|
|
catch (const Exception & e)
|
|
|
|
{
|
|
|
|
throw Exception("Cannot create table from metadata file " + table_metadata_path + ", error: " + e.displayText() +
|
|
|
|
", stack trace:\n" + e.getStackTrace().toString(),
|
|
|
|
ErrorCodes::CANNOT_CREATE_TABLE_FROM_METADATA);
|
|
|
|
}
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-03-23 20:46:43 +00:00
|
|
|
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context)
|
|
|
|
: DatabaseWithOwnTablesBase(std::move(name_))
|
|
|
|
, metadata_path(metadata_path_)
|
|
|
|
, data_path(context.getPath() + "data/" + escapeForFileName(name) + "/")
|
|
|
|
, log(&Logger::get("DatabaseOrdinary (" + name + ")"))
|
2016-06-10 20:46:58 +00:00
|
|
|
{
|
2019-10-02 08:58:28 +00:00
|
|
|
Poco::File(getDataPath()).createDirectories();
|
2016-06-10 20:46:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-09-11 12:39:01 +00:00
|
|
|
void DatabaseOrdinary::loadTables(
|
|
|
|
Context & context,
|
|
|
|
bool has_force_restore_data_flag)
|
2016-03-19 01:18:49 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using FileNames = std::vector<std::string>;
|
|
|
|
FileNames file_names;
|
|
|
|
|
2019-10-03 08:27:43 +00:00
|
|
|
DatabaseOnDisk::iterateTableFiles(*this, log, [&file_names](const String & file_name) {
|
|
|
|
file_names.push_back(file_name);
|
|
|
|
});
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-10-17 02:34:14 +00:00
|
|
|
if (file_names.empty())
|
|
|
|
return;
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/** Tables load faster if they are loaded in sorted (by name) order.
|
2017-06-21 19:07:08 +00:00
|
|
|
* Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
|
2017-04-01 07:20:54 +00:00
|
|
|
* which does not correspond to order tables creation and does not correspond to order of their location on disk.
|
|
|
|
*/
|
|
|
|
std::sort(file_names.begin(), file_names.end());
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
const size_t total_tables = file_names.size();
|
2017-04-01 07:20:54 +00:00
|
|
|
LOG_INFO(log, "Total " << total_tables << " tables.");
|
|
|
|
|
2018-02-25 02:43:27 +00:00
|
|
|
AtomicStopwatch watch;
|
2017-04-01 07:20:54 +00:00
|
|
|
std::atomic<size_t> tables_processed {0};
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
auto loadOneTable = [&](const String & table)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-10-02 08:58:28 +00:00
|
|
|
loadTable(context, getMetadataPath(), *this, getDatabaseName(), getDataPath(), table, has_force_restore_data_flag);
|
2018-12-05 11:20:55 +00:00
|
|
|
|
2018-10-17 01:59:24 +00:00
|
|
|
/// Messages, so that it's not boring to wait for the server to load for a long time.
|
2019-07-19 14:22:57 +00:00
|
|
|
if (++tables_processed % PRINT_MESSAGE_EACH_N_TABLES == 0
|
2018-10-17 01:59:24 +00:00
|
|
|
|| watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-10-17 01:59:24 +00:00
|
|
|
LOG_INFO(log, std::fixed << std::setprecision(2) << tables_processed * 100.0 / total_tables << "%");
|
|
|
|
watch.restart();
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
};
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
ThreadPool pool(SettingMaxThreads().getAutoValue());
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
for (const auto & file_name : file_names)
|
|
|
|
{
|
|
|
|
pool.schedule([&]() { loadOneTable(file_name); });
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
pool.wait();
|
2018-12-05 12:46:32 +00:00
|
|
|
|
2017-06-06 17:06:14 +00:00
|
|
|
/// After all tables was basically initialized, startup them.
|
2019-07-19 14:22:57 +00:00
|
|
|
startupTables(pool);
|
2017-06-06 17:06:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool)
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
|
|
|
LOG_INFO(log, "Starting up tables.");
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
const size_t total_tables = tables.size();
|
2018-10-17 02:34:14 +00:00
|
|
|
if (!total_tables)
|
|
|
|
return;
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
AtomicStopwatch watch;
|
|
|
|
std::atomic<size_t> tables_processed {0};
|
|
|
|
|
|
|
|
auto startupOneTable = [&](const StoragePtr & table)
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
2019-07-19 14:22:57 +00:00
|
|
|
table->startup();
|
2018-12-05 12:46:32 +00:00
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
if (++tables_processed % PRINT_MESSAGE_EACH_N_TABLES == 0
|
2018-10-17 01:59:24 +00:00
|
|
|
|| watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
2018-10-17 01:59:24 +00:00
|
|
|
LOG_INFO(log, std::fixed << std::setprecision(2) << tables_processed * 100.0 / total_tables << "%");
|
2019-07-19 21:02:29 +00:00
|
|
|
watch.restart();
|
2018-10-17 01:59:24 +00:00
|
|
|
}
|
2017-06-06 17:06:14 +00:00
|
|
|
};
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
for (const auto & table : tables)
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
2019-07-19 14:22:57 +00:00
|
|
|
thread_pool.schedule([&]() { startupOneTable(table.second); });
|
2017-06-06 17:06:14 +00:00
|
|
|
}
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
thread_pool.wait();
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-01-23 18:05:07 +00:00
|
|
|
void DatabaseOrdinary::createTable(
|
2017-09-11 12:39:01 +00:00
|
|
|
const Context & context,
|
|
|
|
const String & table_name,
|
|
|
|
const StoragePtr & table,
|
2017-10-25 19:52:32 +00:00
|
|
|
const ASTPtr & query)
|
2016-03-19 01:18:49 +00:00
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
DatabaseOnDisk::createTable(*this, context, table_name, table, query);
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-09-11 12:39:01 +00:00
|
|
|
void DatabaseOrdinary::removeTable(
|
2019-10-03 08:27:43 +00:00
|
|
|
const Context & context,
|
2017-09-11 12:39:01 +00:00
|
|
|
const String & table_name)
|
2016-03-19 01:18:49 +00:00
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
DatabaseOnDisk::removeTable(*this, context, table_name, log);
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void DatabaseOrdinary::renameTable(
|
2017-09-11 12:39:01 +00:00
|
|
|
const Context & context,
|
|
|
|
const String & table_name,
|
|
|
|
IDatabase & to_database,
|
2019-08-27 20:43:08 +00:00
|
|
|
const String & to_table_name,
|
|
|
|
TableStructureWriteLockHolder & lock)
|
2016-03-19 01:18:49 +00:00
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
DatabaseOnDisk::renameTable<DatabaseOrdinary>(*this, context, table_name, to_database, to_table_name, lock);
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-09-11 12:39:01 +00:00
|
|
|
time_t DatabaseOrdinary::getTableMetadataModificationTime(
|
2017-12-01 21:13:25 +00:00
|
|
|
const Context & /*context*/,
|
2017-09-11 12:39:01 +00:00
|
|
|
const String & table_name)
|
2016-09-02 13:04:11 +00:00
|
|
|
{
|
2018-02-22 09:09:14 +00:00
|
|
|
String table_metadata_path = getTableMetadataPath(table_name);
|
2017-04-01 07:20:54 +00:00
|
|
|
Poco::File meta_file(table_metadata_path);
|
|
|
|
|
|
|
|
if (meta_file.exists())
|
|
|
|
{
|
|
|
|
return meta_file.getLastModified().epochTime();
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
return static_cast<time_t>(0);
|
|
|
|
}
|
2016-09-02 13:04:11 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 19:56:44 +00:00
|
|
|
ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
|
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
|
2018-03-14 19:56:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
|
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
|
2018-03-14 19:56:44 +00:00
|
|
|
}
|
|
|
|
|
2019-10-03 08:27:43 +00:00
|
|
|
ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & context) const
|
2018-03-13 13:28:32 +00:00
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
2016-05-13 21:08:19 +00:00
|
|
|
void DatabaseOrdinary::alterTable(
|
2017-04-01 07:20:54 +00:00
|
|
|
const Context & context,
|
2019-01-07 10:40:58 +00:00
|
|
|
const String & table_name,
|
2018-03-06 20:18:34 +00:00
|
|
|
const ColumnsDescription & columns,
|
2019-02-05 14:50:25 +00:00
|
|
|
const IndicesDescription & indices,
|
2019-06-02 14:41:12 +00:00
|
|
|
const ConstraintsDescription & constraints,
|
2017-09-17 18:49:43 +00:00
|
|
|
const ASTModifier & storage_modifier)
|
2016-05-13 21:08:19 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Read the definition of the table and replace the necessary parts with new ones.
|
|
|
|
|
2019-01-07 10:40:58 +00:00
|
|
|
String table_name_escaped = escapeForFileName(table_name);
|
2019-10-02 08:58:28 +00:00
|
|
|
String table_metadata_tmp_path = getMetadataPath() + "/" + table_name_escaped + ".sql.tmp";
|
|
|
|
String table_metadata_path = getMetadataPath() + "/" + table_name_escaped + ".sql";
|
2017-04-01 07:20:54 +00:00
|
|
|
String statement;
|
|
|
|
|
|
|
|
{
|
|
|
|
char in_buf[METADATA_FILE_BUFFER_SIZE];
|
|
|
|
ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
|
|
|
|
readStringUntilEOF(statement, in);
|
|
|
|
}
|
|
|
|
|
|
|
|
ParserCreateQuery parser;
|
2018-04-16 15:11:13 +00:00
|
|
|
ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-03-15 16:14:13 +00:00
|
|
|
const auto & ast_create_query = ast->as<ASTCreateQuery &>();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-03-06 20:18:34 +00:00
|
|
|
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns);
|
2019-02-05 14:50:25 +00:00
|
|
|
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(indices);
|
2019-06-02 14:41:12 +00:00
|
|
|
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(constraints);
|
2019-02-05 14:50:25 +00:00
|
|
|
|
2019-03-15 16:14:13 +00:00
|
|
|
ast_create_query.columns_list->replace(ast_create_query.columns_list->columns, new_columns);
|
2019-02-05 14:50:25 +00:00
|
|
|
|
2019-03-15 16:14:13 +00:00
|
|
|
if (ast_create_query.columns_list->indices)
|
|
|
|
ast_create_query.columns_list->replace(ast_create_query.columns_list->indices, new_indices);
|
2019-02-05 14:50:25 +00:00
|
|
|
else
|
2019-03-15 16:14:13 +00:00
|
|
|
ast_create_query.columns_list->set(ast_create_query.columns_list->indices, new_indices);
|
2017-09-17 18:49:43 +00:00
|
|
|
|
2019-06-02 14:41:12 +00:00
|
|
|
if (ast_create_query.columns_list->constraints)
|
|
|
|
ast_create_query.columns_list->replace(ast_create_query.columns_list->constraints, new_constraints);
|
|
|
|
else
|
|
|
|
ast_create_query.columns_list->set(ast_create_query.columns_list->constraints, new_constraints);
|
|
|
|
|
2017-09-17 18:49:43 +00:00
|
|
|
if (storage_modifier)
|
2019-03-15 16:14:13 +00:00
|
|
|
storage_modifier(*ast_create_query.storage);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
statement = getTableDefinitionFromCreateQuery(ast);
|
|
|
|
|
|
|
|
{
|
|
|
|
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
|
|
|
|
writeString(statement, out);
|
|
|
|
out.next();
|
|
|
|
if (context.getSettingsRef().fsync_metadata)
|
|
|
|
out.sync();
|
|
|
|
out.close();
|
|
|
|
}
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
/// rename atomically replaces the old file with the new one.
|
|
|
|
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
|
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
|
|
|
Poco::File(table_metadata_tmp_path).remove();
|
|
|
|
throw;
|
|
|
|
}
|
2016-05-13 21:08:19 +00:00
|
|
|
}
|
|
|
|
|
2018-06-09 15:48:22 +00:00
|
|
|
|
|
|
|
void DatabaseOrdinary::drop()
|
|
|
|
{
|
2019-10-03 08:27:43 +00:00
|
|
|
DatabaseOnDisk::drop(*this);
|
2018-06-09 15:48:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-02-21 19:26:59 +00:00
|
|
|
String DatabaseOrdinary::getDataPath() const
|
2017-11-03 19:53:10 +00:00
|
|
|
{
|
|
|
|
return data_path;
|
|
|
|
}
|
|
|
|
|
2018-02-21 19:26:59 +00:00
|
|
|
String DatabaseOrdinary::getMetadataPath() const
|
|
|
|
{
|
|
|
|
return metadata_path;
|
|
|
|
}
|
|
|
|
|
2018-05-10 15:56:38 +00:00
|
|
|
String DatabaseOrdinary::getDatabaseName() const
|
|
|
|
{
|
|
|
|
return name;
|
|
|
|
}
|
|
|
|
|
2018-02-21 19:26:59 +00:00
|
|
|
String DatabaseOrdinary::getTableMetadataPath(const String & table_name) const
|
|
|
|
{
|
2019-10-02 08:58:28 +00:00
|
|
|
return detail::getTableMetadataPath(getMetadataPath(), table_name);
|
2018-02-21 19:26:59 +00:00
|
|
|
}
|
|
|
|
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|