ClickHouse/dbms/src/Interpreters/loadMetadata.cpp

146 lines
4.2 KiB
C++
Raw Normal View History

#include <iomanip>
#include <thread>
#include <future>
#include <common/ThreadPool.h>
2011-10-30 11:30:52 +00:00
#include <Poco/DirectoryIterator.h>
#include <Poco/FileStream.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/parseQuery.h>
2011-10-30 11:30:52 +00:00
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/Context.h>
#include <Interpreters/loadMetadata.h>
2011-10-30 11:30:52 +00:00
#include <Databases/DatabaseOrdinary.h>
#include <IO/ReadBufferFromFile.h>
#include <Common/escapeForFileName.h>
#include <Common/Stopwatch.h>
2011-10-30 11:30:52 +00:00
namespace DB
{
Squashed commit of the following: commit f9b478181cd49224154cc350fb57df7121842f1c Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Sat Mar 19 04:06:36 2016 +0300 Database engines: development [#METR-19997]. commit f7a10a67761ccfd05f3dac32d6444920cd8d4d60 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Sat Mar 19 03:44:37 2016 +0300 Database engines: development [#METR-19997]. commit bd98a8558e98bad2bed278e5762c4e0fc66e6f38 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Sat Mar 19 00:33:59 2016 +0300 Database engines: development [#METR-19997]. commit 19712fd884c22a4e2c2b67474086dea8f44e7c7b Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Sat Mar 19 00:03:11 2016 +0300 Database engines: development [#METR-19997]. commit 50274d6df7e91fcc34aab8a8c72347daa2c6512f Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Fri Mar 18 23:24:57 2016 +0300 Database engines: development [#METR-19997]. commit 4a0b99b19b34e90ef8b7be2d199f6232e36ef3f7 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Fri Mar 18 22:50:36 2016 +0300 Database engines: development [#METR-19997]. commit 44ff3ebba7a3e460a27a89f31ddf199dbea1d182 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Fri Mar 18 15:09:17 2016 +0300 Database engines: development [#METR-19997]. commit 137c31f3004cfd282473b6acb01cbe1b4ca2aadd Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Fri Mar 18 03:26:34 2016 +0300 Database engines: development [#METR-19997]. commit aa4c0496d4afe4a691164254be2bd5600542b38a Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Fri Mar 18 03:22:59 2016 +0300 Database engines: development [#METR-19997]. commit 5a94d1f0607450a2dac28a4d7df8b1393a864c23 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Fri Mar 18 01:02:40 2016 +0300 Database engines: development [#METR-19997]. commit 50fd5b52ea1141955a5dfba0dcb191f3289ac25b Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Thu Mar 17 23:23:40 2016 +0300 Database engines: development [#METR-19997]. commit a333d91b058e4f56dd83a6d2878c3c2bd8efc002 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Thu Mar 17 20:29:07 2016 +0300 Database engines: development [#METR-19997]. commit f81d366e7ac8348436f2698d040f8e341743a024 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Thu Mar 17 01:30:23 2016 +0300 Database engines: development [#METR-19997]. commit d0696860c9060827896214c08d147c759ea79376 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Wed Mar 16 21:55:31 2016 +0300 Database engines: development [#METR-19997]. commit 46a168c2ada140a0e95cd8d4b9d8ba9bac855d11 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Wed Mar 16 08:00:58 2016 +0300 Database engines: development [#METR-19997]. commit 20a2bad161454225fc1b5f9b919b842fbebc3231 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Wed Mar 16 06:51:10 2016 +0300 Database engines: development [#METR-19997]. commit ca0a77fcc2a8d0b276eb3743c53551ad3fe16314 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Wed Mar 16 06:02:20 2016 +0300 Reverted erroneous modification [#METR-19997]. commit 1370bdcc4594182f6ef2b146f9afabfe1c295080 Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Wed Mar 16 00:41:34 2016 +0300 Database engines: development [#METR-19997]. commit 16e72c67041cae6471509d3f0f3d4a9aa7b7dc0f Author: Alexey Milovidov <milovidov@yandex-team.ru> Date: Tue Mar 15 00:41:48 2016 +0300 Database engines: development [#METR-19997].
2016-03-19 01:18:49 +00:00
static void executeCreateQuery(
const String & query,
Context & context,
const String & database,
const String & file_name,
2017-06-18 05:43:29 +00:00
ThreadPool * pool,
bool has_force_restore_data_flag)
2011-10-30 11:30:52 +00:00
{
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "in file " + file_name);
2011-10-30 11:30:52 +00:00
ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
ast_create_query.attach = true;
ast_create_query.database = database;
2012-03-09 03:06:09 +00:00
InterpreterCreateQuery interpreter(ast, context);
interpreter.setInternal(true);
2017-06-18 05:43:29 +00:00
if (pool)
interpreter.setDatabaseLoadingThreadpool(*pool);
interpreter.setForceRestoreData(has_force_restore_data_flag);
interpreter.execute();
}
2017-06-18 05:43:29 +00:00
static void loadDatabase(
Context & context,
const String & database,
const String & database_path,
ThreadPool * thread_pool,
bool force_restore_data)
{
/// There may exist .sql file with database creation statement.
/// Or, if it is absent, then database with default engine is created.
String database_attach_query;
String database_metadata_file = database_path + ".sql";
if (Poco::File(database_metadata_file).exists())
{
ReadBufferFromFile in(database_metadata_file, 1024);
readStringUntilEOF(database_attach_query, in);
}
else
database_attach_query = "ATTACH DATABASE " + backQuoteIfNeed(database);
executeCreateQuery(database_attach_query, context, database, database_metadata_file, thread_pool, force_restore_data);
}
#define SYSTEM_DATABASE "system"
2011-10-30 11:30:52 +00:00
void loadMetadata(Context & context)
{
String path = context.getPath() + "metadata";
2011-10-30 11:30:52 +00:00
/** There may exist 'force_restore_data' file, that means,
* skip safety threshold on difference of data parts while initializing tables.
* This file is deleted after successful loading of tables.
* (flag is "one-shot")
*/
Poco::File force_restore_data_flag_file(context.getFlagsPath() + "force_restore_data");
bool has_force_restore_data_flag = force_restore_data_flag_file.exists();
/// For parallel tables loading.
ThreadPool thread_pool(SettingMaxThreads().getAutoValue());
2015-09-29 20:47:15 +00:00
/// Loop over databases.
std::map<String, String> databases;
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator it(path); it != dir_end; ++it)
{
if (!it->isDirectory())
continue;
2011-10-30 11:30:52 +00:00
/// For '.svn', '.gitignore' directory and similar.
if (it.name().at(0) == '.')
continue;
2012-03-09 03:56:12 +00:00
2017-06-18 05:43:29 +00:00
if (it.name() == SYSTEM_DATABASE)
continue;
databases.emplace(unescapeForFileName(it.name()), it.path().toString());
}
2017-06-18 05:43:29 +00:00
for (const auto & elem : databases)
loadDatabase(context, elem.first, elem.second, &thread_pool, has_force_restore_data_flag);
2017-06-18 05:43:29 +00:00
thread_pool.wait();
2017-06-18 05:43:29 +00:00
if (has_force_restore_data_flag)
force_restore_data_flag_file.remove();
}
2017-06-18 05:43:29 +00:00
void loadMetadataSystem(Context & context)
{
String path = context.getPath() + "metadata/" SYSTEM_DATABASE;
if (Poco::File(path).exists())
{
2017-06-18 05:43:29 +00:00
/// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted.
loadDatabase(context, SYSTEM_DATABASE, path, nullptr, true);
}
else
{
/// Initialize system database manually
String global_path = context.getPath();
2017-06-18 05:43:29 +00:00
Poco::File(global_path + "data/" SYSTEM_DATABASE).createDirectories();
Poco::File(global_path + "metadata/" SYSTEM_DATABASE).createDirectories();
auto system_database = std::make_shared<DatabaseOrdinary>(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE, context);
2017-06-18 05:43:29 +00:00
context.addDatabase(SYSTEM_DATABASE, system_database);
}
2011-10-30 11:30:52 +00:00
}
}