2018-06-05 19:46:49 +00:00
|
|
|
#include <iomanip>
|
|
|
|
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Core/Settings.h>
|
2019-10-02 12:58:13 +00:00
|
|
|
#include <Databases/DatabaseOnDisk.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Databases/DatabaseOrdinary.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Databases/DatabasesCommon.h>
|
|
|
|
#include <IO/ReadBufferFromFile.h>
|
2018-06-05 19:46:49 +00:00
|
|
|
#include <IO/ReadHelpers.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <IO/WriteBufferFromFile.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Interpreters/InterpreterCreateQuery.h>
|
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/ParserCreateQuery.h>
|
2019-10-10 17:33:01 +00:00
|
|
|
#include <Storages/StorageFactory.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Parsers/parseQuery.h>
|
2019-10-10 17:33:01 +00:00
|
|
|
#include <Parsers/formatAST.h>
|
2019-12-29 01:08:12 +00:00
|
|
|
#include <Parsers/ASTSetQuery.h>
|
2019-10-10 17:33:01 +00:00
|
|
|
#include <TableFunctions/TableFunctionFactory.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
|
2019-12-27 14:36:59 +00:00
|
|
|
#include <Parsers/queryToString.h>
|
|
|
|
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Poco/DirectoryIterator.h>
|
|
|
|
#include <Poco/Event.h>
|
|
|
|
#include <Common/Stopwatch.h>
|
2019-11-25 22:48:23 +00:00
|
|
|
#include <Common/quoteString.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Common/ThreadPool.h>
|
|
|
|
#include <Common/escapeForFileName.h>
|
|
|
|
#include <Common/typeid_cast.h>
|
|
|
|
#include <common/logger_useful.h>
|
2018-12-05 11:20:55 +00:00
|
|
|
#include <ext/scope_guard.h>
|
2016-03-19 01:18:49 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2019-10-10 17:33:01 +00:00
|
|
|
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA;
|
|
|
|
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
|
|
|
|
extern const int CANNOT_PARSE_TEXT;
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256;
|
2016-03-19 01:18:49 +00:00
|
|
|
static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5;
|
|
|
|
static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
|
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
|
2019-12-12 16:41:41 +00:00
|
|
|
namespace
|
2019-10-10 17:33:01 +00:00
|
|
|
{
|
2019-12-12 16:41:41 +00:00
|
|
|
void tryAttachTable(
|
|
|
|
Context & context,
|
|
|
|
const ASTCreateQuery & query,
|
|
|
|
DatabaseOrdinary & database,
|
|
|
|
const String & database_name,
|
|
|
|
bool has_force_restore_data_flag)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-12-12 16:41:41 +00:00
|
|
|
assert(!query.is_dictionary);
|
|
|
|
try
|
|
|
|
{
|
|
|
|
String table_name;
|
|
|
|
StoragePtr table;
|
|
|
|
std::tie(table_name, table)
|
2019-12-25 16:13:48 +00:00
|
|
|
= createTableFromAST(query, database_name, database.getTableDataPath(query), context, has_force_restore_data_flag);
|
2019-12-12 16:41:41 +00:00
|
|
|
database.attachTable(table_name, table);
|
|
|
|
}
|
2020-01-02 07:16:02 +00:00
|
|
|
catch (Exception & e)
|
2019-12-12 16:41:41 +00:00
|
|
|
{
|
2020-01-02 07:16:02 +00:00
|
|
|
e.addMessage("Cannot attach table '" + backQuote(query.table) + "' from query " + serializeAST(query));
|
|
|
|
throw;
|
2019-12-12 16:41:41 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2019-12-12 16:41:41 +00:00
|
|
|
|
|
|
|
|
|
|
|
void tryAttachDictionary(
|
|
|
|
Context & context,
|
|
|
|
const ASTCreateQuery & query,
|
|
|
|
DatabaseOrdinary & database)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-12-12 16:41:41 +00:00
|
|
|
assert(query.is_dictionary);
|
|
|
|
try
|
|
|
|
{
|
|
|
|
database.attachDictionary(query.table, context);
|
|
|
|
}
|
2020-01-02 07:16:02 +00:00
|
|
|
catch (Exception & e)
|
2019-12-12 16:41:41 +00:00
|
|
|
{
|
2020-01-02 07:16:02 +00:00
|
|
|
e.addMessage("Cannot attach table '" + backQuote(query.table) + "' from query " + serializeAST(query));
|
|
|
|
throw;
|
2019-12-12 16:41:41 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2019-10-10 17:33:01 +00:00
|
|
|
|
|
|
|
|
2019-12-12 16:41:41 +00:00
|
|
|
void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-12-12 16:41:41 +00:00
|
|
|
if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))
|
|
|
|
{
|
|
|
|
LOG_INFO(log, std::fixed << std::setprecision(2) << processed * 100.0 / total << "%");
|
|
|
|
watch.restart();
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-11-06 16:05:04 +00:00
|
|
|
DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context_)
|
2019-12-25 16:13:48 +00:00
|
|
|
: DatabaseWithDictionaries(name_, metadata_path_, "DatabaseOrdinary (" + name_ + ")")
|
2016-06-10 20:46:58 +00:00
|
|
|
{
|
2019-10-25 19:07:47 +00:00
|
|
|
Poco::File(context_.getPath() + getDataPath()).createDirectories();
|
2016-06-10 20:46:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
void DatabaseOrdinary::loadStoredObjects(
|
2017-09-11 12:39:01 +00:00
|
|
|
Context & context,
|
|
|
|
bool has_force_restore_data_flag)
|
2016-03-19 01:18:49 +00:00
|
|
|
{
|
2019-10-16 17:06:52 +00:00
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
/** Tables load faster if they are loaded in sorted (by name) order.
|
|
|
|
* Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
|
|
|
|
* which does not correspond to order tables creation and does not correspond to order of their location on disk.
|
|
|
|
*/
|
2019-10-22 10:47:43 +00:00
|
|
|
using FileNames = std::map<std::string, ASTPtr>;
|
2017-04-01 07:20:54 +00:00
|
|
|
FileNames file_names;
|
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
size_t total_dictionaries = 0;
|
2019-11-06 16:05:04 +00:00
|
|
|
iterateMetadataFiles(context, [&file_names, &total_dictionaries, this](const String & file_name)
|
2019-10-05 10:21:04 +00:00
|
|
|
{
|
2019-12-25 16:13:48 +00:00
|
|
|
String full_path = getMetadataPath() + file_name;
|
2019-10-10 17:33:01 +00:00
|
|
|
try
|
|
|
|
{
|
2019-12-25 13:58:15 +00:00
|
|
|
auto ast = parseQueryFromMetadata(full_path, /*throw_on_error*/ true, /*remove_empty*/false);
|
2019-10-10 17:33:01 +00:00
|
|
|
if (ast)
|
|
|
|
{
|
2019-10-22 10:47:43 +00:00
|
|
|
auto * create_query = ast->as<ASTCreateQuery>();
|
|
|
|
file_names[file_name] = ast;
|
|
|
|
total_dictionaries += create_query->is_dictionary;
|
2019-10-10 17:33:01 +00:00
|
|
|
}
|
|
|
|
}
|
2020-01-02 07:16:02 +00:00
|
|
|
catch (Exception & e)
|
2019-10-10 17:33:01 +00:00
|
|
|
{
|
2020-01-02 07:16:02 +00:00
|
|
|
e.addMessage("Cannot parse definition from metadata file " + full_path);
|
|
|
|
throw;
|
2019-10-10 17:33:01 +00:00
|
|
|
}
|
|
|
|
|
2019-10-03 08:27:43 +00:00
|
|
|
});
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
size_t total_tables = file_names.size() - total_dictionaries;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-10-10 17:33:01 +00:00
|
|
|
LOG_INFO(log, "Total " << total_tables << " tables and " << total_dictionaries << " dictionaries.");
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-02-25 02:43:27 +00:00
|
|
|
AtomicStopwatch watch;
|
2019-10-10 17:33:01 +00:00
|
|
|
std::atomic<size_t> tables_processed{0};
|
|
|
|
std::atomic<size_t> dictionaries_processed{0};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
ThreadPool pool(SettingMaxThreads().getAutoValue());
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-12 16:41:41 +00:00
|
|
|
/// Attach tables.
|
2019-10-21 08:46:11 +00:00
|
|
|
for (const auto & name_with_query : file_names)
|
2019-07-19 14:22:57 +00:00
|
|
|
{
|
2019-12-12 16:41:41 +00:00
|
|
|
const auto & create_query = name_with_query.second->as<const ASTCreateQuery &>();
|
|
|
|
if (!create_query.is_dictionary)
|
|
|
|
pool.scheduleOrThrowOnError([&]()
|
|
|
|
{
|
2019-10-30 14:17:55 +00:00
|
|
|
tryAttachTable(context, create_query, *this, getDatabaseName(), has_force_restore_data_flag);
|
2019-12-12 16:41:41 +00:00
|
|
|
|
|
|
|
/// Messages, so that it's not boring to wait for the server to load for a long time.
|
|
|
|
logAboutProgress(log, ++tables_processed, total_tables, watch);
|
|
|
|
});
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
pool.wait();
|
2018-12-05 12:46:32 +00:00
|
|
|
|
2017-06-06 17:06:14 +00:00
|
|
|
/// After all tables was basically initialized, startup them.
|
2019-07-19 14:22:57 +00:00
|
|
|
startupTables(pool);
|
2019-10-17 17:18:54 +00:00
|
|
|
|
2019-12-12 16:41:41 +00:00
|
|
|
/// Attach dictionaries.
|
2019-12-30 23:30:06 +00:00
|
|
|
attachToExternalDictionariesLoader(context);
|
2019-12-12 16:41:41 +00:00
|
|
|
for (const auto & name_with_query : file_names)
|
2019-11-25 22:48:23 +00:00
|
|
|
{
|
2019-12-12 16:41:41 +00:00
|
|
|
auto create_query = name_with_query.second->as<const ASTCreateQuery &>();
|
|
|
|
if (create_query.is_dictionary)
|
|
|
|
{
|
|
|
|
tryAttachDictionary(context, create_query, *this);
|
|
|
|
|
|
|
|
/// Messages, so that it's not boring to wait for the server to load for a long time.
|
|
|
|
logAboutProgress(log, ++dictionaries_processed, total_dictionaries, watch);
|
|
|
|
}
|
|
|
|
}
|
2017-06-06 17:06:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool)
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
|
|
|
LOG_INFO(log, "Starting up tables.");
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
const size_t total_tables = tables.size();
|
2018-10-17 02:34:14 +00:00
|
|
|
if (!total_tables)
|
|
|
|
return;
|
|
|
|
|
2019-07-19 14:22:57 +00:00
|
|
|
AtomicStopwatch watch;
|
2019-10-10 17:33:01 +00:00
|
|
|
std::atomic<size_t> tables_processed{0};
|
2019-07-19 14:22:57 +00:00
|
|
|
|
|
|
|
auto startupOneTable = [&](const StoragePtr & table)
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
2019-07-19 14:22:57 +00:00
|
|
|
table->startup();
|
2019-10-10 17:33:01 +00:00
|
|
|
logAboutProgress(log, ++tables_processed, total_tables, watch);
|
2017-06-06 17:06:14 +00:00
|
|
|
};
|
|
|
|
|
2019-10-17 14:41:27 +00:00
|
|
|
try
|
2017-06-06 17:06:14 +00:00
|
|
|
{
|
2019-10-17 14:41:27 +00:00
|
|
|
for (const auto & table : tables)
|
|
|
|
thread_pool.scheduleOrThrowOnError([&]() { startupOneTable(table.second); });
|
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
|
|
|
thread_pool.wait();
|
|
|
|
throw;
|
2017-06-06 17:06:14 +00:00
|
|
|
}
|
2019-07-19 14:22:57 +00:00
|
|
|
thread_pool.wait();
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|
|
|
|
|
2016-05-13 21:08:19 +00:00
|
|
|
void DatabaseOrdinary::alterTable(
|
2017-04-01 07:20:54 +00:00
|
|
|
const Context & context,
|
2019-01-07 10:40:58 +00:00
|
|
|
const String & table_name,
|
2019-12-26 18:17:05 +00:00
|
|
|
const StorageInMemoryMetadata & metadata)
|
2016-05-13 21:08:19 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Read the definition of the table and replace the necessary parts with new ones.
|
2019-12-25 16:13:48 +00:00
|
|
|
String table_metadata_path = getObjectMetadataPath(table_name);
|
|
|
|
String table_metadata_tmp_path = table_metadata_path + ".tmp";
|
2017-04-01 07:20:54 +00:00
|
|
|
String statement;
|
|
|
|
|
|
|
|
{
|
|
|
|
char in_buf[METADATA_FILE_BUFFER_SIZE];
|
|
|
|
ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
|
|
|
|
readStringUntilEOF(statement, in);
|
|
|
|
}
|
|
|
|
|
|
|
|
ParserCreateQuery parser;
|
2018-04-16 15:11:13 +00:00
|
|
|
ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-03-15 16:14:13 +00:00
|
|
|
const auto & ast_create_query = ast->as<ASTCreateQuery &>();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-26 18:17:05 +00:00
|
|
|
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
|
|
|
|
ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.indices);
|
|
|
|
ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(metadata.constraints);
|
2019-02-05 14:50:25 +00:00
|
|
|
|
2019-03-15 16:14:13 +00:00
|
|
|
ast_create_query.columns_list->replace(ast_create_query.columns_list->columns, new_columns);
|
2019-12-16 18:10:35 +00:00
|
|
|
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices);
|
|
|
|
ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints);
|
2019-06-02 14:41:12 +00:00
|
|
|
|
2019-12-26 18:17:05 +00:00
|
|
|
ASTStorage & storage_ast = *ast_create_query.storage;
|
2019-12-27 14:36:59 +00:00
|
|
|
/// ORDER BY may change, but cannot appear, it's required construction
|
2019-12-27 14:46:11 +00:00
|
|
|
if (metadata.order_by_ast && storage_ast.order_by)
|
|
|
|
storage_ast.set(storage_ast.order_by, metadata.order_by_ast);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-27 14:46:11 +00:00
|
|
|
if (metadata.primary_key_ast)
|
|
|
|
storage_ast.set(storage_ast.primary_key, metadata.primary_key_ast);
|
2019-12-26 18:17:05 +00:00
|
|
|
|
2019-12-27 14:46:11 +00:00
|
|
|
if (metadata.ttl_for_table_ast)
|
|
|
|
storage_ast.set(storage_ast.ttl_table, metadata.ttl_for_table_ast);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-27 14:36:59 +00:00
|
|
|
if (metadata.settings_ast)
|
|
|
|
storage_ast.set(storage_ast.settings, metadata.settings_ast);
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-27 14:36:59 +00:00
|
|
|
statement = getObjectDefinitionFromCreateQuery(ast);
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
|
|
|
|
writeString(statement, out);
|
|
|
|
out.next();
|
|
|
|
if (context.getSettingsRef().fsync_metadata)
|
|
|
|
out.sync();
|
|
|
|
out.close();
|
|
|
|
}
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
/// rename atomically replaces the old file with the new one.
|
|
|
|
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
|
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
|
|
|
Poco::File(table_metadata_tmp_path).remove();
|
|
|
|
throw;
|
|
|
|
}
|
2016-05-13 21:08:19 +00:00
|
|
|
}
|
|
|
|
|
2016-03-19 01:18:49 +00:00
|
|
|
}
|