2019-01-11 19:12:36 +00:00
|
|
|
#include <Common/ThreadPool.h>
|
2013-06-15 06:16:17 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Parsers/ParserCreateQuery.h>
|
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/parseQuery.h>
|
2011-10-30 11:30:52 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Interpreters/InterpreterCreateQuery.h>
|
2022-09-09 16:27:19 +00:00
|
|
|
#include <Interpreters/InterpreterSystemQuery.h>
|
2017-05-23 17:35:05 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Interpreters/loadMetadata.h>
|
2022-06-23 19:38:43 +00:00
|
|
|
#include <Interpreters/executeQuery.h>
|
2011-10-30 11:30:52 +00:00
|
|
|
|
2017-06-15 20:08:26 +00:00
|
|
|
#include <Databases/DatabaseOrdinary.h>
|
2021-08-26 13:19:52 +00:00
|
|
|
#include <Databases/TablesLoader.h>
|
2022-08-05 19:41:02 +00:00
|
|
|
#include <Storages/StorageMaterializedView.h>
|
2017-06-15 20:08:26 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/ReadBufferFromFile.h>
|
2018-06-05 19:46:49 +00:00
|
|
|
#include <IO/ReadHelpers.h>
|
2013-01-17 19:36:34 +00:00
|
|
|
|
2023-03-22 07:49:22 +00:00
|
|
|
#include <Common/escapeForFileName.h>
|
2019-01-14 18:15:04 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2022-04-27 15:05:45 +00:00
|
|
|
#include <Common/logger_useful.h>
|
2023-03-22 07:49:22 +00:00
|
|
|
#include <Common/CurrentMetrics.h>
|
|
|
|
|
|
|
|
#include <filesystem>
|
2013-06-15 06:14:41 +00:00
|
|
|
|
2023-03-13 16:13:49 +00:00
|
|
|
#define ORDINARY_TO_ATOMIC_PREFIX ".tmp_convert."
|
|
|
|
|
2021-04-28 10:42:07 +00:00
|
|
|
namespace fs = std::filesystem;
|
2011-10-30 11:30:52 +00:00
|
|
|
|
2023-03-22 07:49:22 +00:00
|
|
|
namespace CurrentMetrics
|
|
|
|
{
|
|
|
|
extern const Metric StartupSystemTablesThreads;
|
|
|
|
extern const Metric StartupSystemTablesThreadsActive;
|
2023-11-18 18:07:59 +00:00
|
|
|
extern const Metric StartupSystemTablesThreadsScheduled;
|
2023-03-22 07:49:22 +00:00
|
|
|
}
|
|
|
|
|
2011-10-30 11:30:52 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int NOT_IMPLEMENTED;
|
2022-06-27 14:10:00 +00:00
|
|
|
extern const int LOGICAL_ERROR;
|
2022-06-23 19:38:43 +00:00
|
|
|
}
|
|
|
|
|
2022-09-09 16:27:19 +00:00
|
|
|
namespace ActionLocks
|
|
|
|
{
|
2023-08-03 10:34:43 +00:00
|
|
|
extern const StorageActionBlockType PartsMerge;
|
|
|
|
extern const StorageActionBlockType PartsFetch;
|
|
|
|
extern const StorageActionBlockType PartsSend;
|
|
|
|
extern const StorageActionBlockType DistributedSend;
|
2022-09-09 16:27:19 +00:00
|
|
|
}
|
|
|
|
|
2016-03-19 01:18:49 +00:00
|
|
|
static void executeCreateQuery(
|
|
|
|
const String & query,
|
2021-05-31 14:49:02 +00:00
|
|
|
ContextMutablePtr context,
|
2016-03-19 01:18:49 +00:00
|
|
|
const String & database,
|
|
|
|
const String & file_name,
|
2022-07-20 20:54:43 +00:00
|
|
|
bool create,
|
2016-08-09 21:48:05 +00:00
|
|
|
bool has_force_restore_data_flag)
|
2011-10-30 11:30:52 +00:00
|
|
|
{
|
|
|
|
ParserCreateQuery parser;
|
2021-04-10 23:33:54 +00:00
|
|
|
ASTPtr ast = parseQuery(
|
|
|
|
parser, query.data(), query.data() + query.size(), "in file " + file_name, 0, context->getSettingsRef().max_parser_depth);
|
2011-10-30 11:30:52 +00:00
|
|
|
|
2019-03-15 16:14:13 +00:00
|
|
|
auto & ast_create_query = ast->as<ASTCreateQuery &>();
|
2021-09-06 22:13:54 +00:00
|
|
|
ast_create_query.setDatabase(database);
|
2012-03-09 03:06:09 +00:00
|
|
|
|
2016-03-19 01:18:49 +00:00
|
|
|
InterpreterCreateQuery interpreter(ast, context);
|
2018-01-18 23:40:32 +00:00
|
|
|
interpreter.setInternal(true);
|
2022-07-20 20:54:43 +00:00
|
|
|
if (!create)
|
|
|
|
{
|
|
|
|
interpreter.setForceAttach(true);
|
|
|
|
interpreter.setForceRestoreData(has_force_restore_data_flag);
|
|
|
|
}
|
2021-09-01 19:42:49 +00:00
|
|
|
interpreter.setLoadDatabaseWithoutTables(true);
|
2016-03-19 01:18:49 +00:00
|
|
|
interpreter.execute();
|
2015-12-13 14:29:58 +00:00
|
|
|
}
|
|
|
|
|
2021-09-09 09:37:51 +00:00
|
|
|
static bool isSystemOrInformationSchema(const String & database_name)
|
|
|
|
{
|
|
|
|
return database_name == DatabaseCatalog::SYSTEM_DATABASE ||
|
|
|
|
database_name == DatabaseCatalog::INFORMATION_SCHEMA ||
|
|
|
|
database_name == DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE;
|
|
|
|
}
|
2015-12-13 14:29:58 +00:00
|
|
|
|
2017-06-18 05:43:29 +00:00
|
|
|
static void loadDatabase(
|
2021-05-31 14:49:02 +00:00
|
|
|
ContextMutablePtr context,
|
2017-06-18 05:43:29 +00:00
|
|
|
const String & database,
|
|
|
|
const String & database_path,
|
|
|
|
bool force_restore_data)
|
|
|
|
{
|
|
|
|
String database_attach_query;
|
|
|
|
String database_metadata_file = database_path + ".sql";
|
|
|
|
|
2021-04-28 10:42:07 +00:00
|
|
|
if (fs::exists(fs::path(database_metadata_file)))
|
2017-06-18 05:43:29 +00:00
|
|
|
{
|
2020-03-17 23:51:35 +00:00
|
|
|
/// There is .sql file with database creation statement.
|
2017-06-18 05:43:29 +00:00
|
|
|
ReadBufferFromFile in(database_metadata_file, 1024);
|
|
|
|
readStringUntilEOF(database_attach_query, in);
|
|
|
|
}
|
|
|
|
else
|
2019-11-28 19:40:51 +00:00
|
|
|
{
|
|
|
|
/// It's first server run and we need create default and system databases.
|
|
|
|
/// .sql file with database engine will be written for CREATE query.
|
|
|
|
database_attach_query = "CREATE DATABASE " + backQuoteIfNeed(database);
|
|
|
|
}
|
2017-06-18 05:43:29 +00:00
|
|
|
|
2020-08-10 01:06:06 +00:00
|
|
|
try
|
|
|
|
{
|
2022-07-20 20:54:43 +00:00
|
|
|
executeCreateQuery(database_attach_query, context, database, database_metadata_file, /* create */ true, force_restore_data);
|
2020-08-10 01:06:06 +00:00
|
|
|
}
|
|
|
|
catch (Exception & e)
|
|
|
|
{
|
2020-08-10 02:58:08 +00:00
|
|
|
e.addMessage(fmt::format("while loading database {} from path {}", backQuote(database), database_path));
|
2020-08-10 01:06:06 +00:00
|
|
|
throw;
|
|
|
|
}
|
2017-06-18 05:43:29 +00:00
|
|
|
}
|
|
|
|
|
2022-06-24 17:10:33 +00:00
|
|
|
static void checkUnsupportedVersion(ContextMutablePtr context, const String & database_name)
|
|
|
|
{
|
|
|
|
/// Produce better exception message
|
|
|
|
String metadata_path = context->getPath() + "metadata/" + database_name;
|
|
|
|
if (fs::exists(fs::path(metadata_path)))
|
|
|
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Data directory for {} database exists, but metadata file does not. "
|
|
|
|
"Probably you are trying to upgrade from version older than 20.7. "
|
|
|
|
"If so, you should upgrade through intermediate version.", database_name);
|
|
|
|
}
|
2017-06-18 05:43:29 +00:00
|
|
|
|
2023-03-13 16:13:49 +00:00
|
|
|
static void checkIncompleteOrdinaryToAtomicConversion(ContextPtr context, const std::map<String, String> & databases)
|
|
|
|
{
|
|
|
|
if (context->getConfigRef().has("allow_reserved_database_name_tmp_convert"))
|
|
|
|
return;
|
|
|
|
|
|
|
|
auto convert_flag_path = fs::path(context->getFlagsPath()) / "convert_ordinary_to_atomic";
|
|
|
|
if (!fs::exists(convert_flag_path))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/// Flag exists. Let's check if we had an unsuccessful conversion attempt previously
|
|
|
|
for (const auto & db : databases)
|
|
|
|
{
|
|
|
|
if (!db.first.starts_with(ORDINARY_TO_ATOMIC_PREFIX))
|
|
|
|
continue;
|
|
|
|
size_t last_dot = db.first.rfind('.');
|
|
|
|
if (last_dot <= strlen(ORDINARY_TO_ATOMIC_PREFIX))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
String actual_name = db.first.substr(strlen(ORDINARY_TO_ATOMIC_PREFIX), last_dot - strlen(ORDINARY_TO_ATOMIC_PREFIX));
|
|
|
|
|
|
|
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Found a database with special name: {}. "
|
|
|
|
"Most likely it indicates that conversion of database {} from Ordinary to Atomic "
|
|
|
|
"was interrupted or failed in the middle. You can add <allow_reserved_database_name_tmp_convert> to config.xml "
|
|
|
|
"or remove convert_ordinary_to_atomic file from flags/ directory, so the server will start forcefully. "
|
|
|
|
"After starting the server, you can finish conversion manually by moving rest of the tables from {} to {} "
|
|
|
|
"(using RENAME TABLE) and executing DROP DATABASE {} and RENAME DATABASE {} TO {}",
|
|
|
|
backQuote(db.first), backQuote(actual_name), backQuote(actual_name), backQuote(db.first),
|
|
|
|
backQuote(actual_name), backQuote(db.first), backQuote(actual_name));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-31 14:49:02 +00:00
|
|
|
void loadMetadata(ContextMutablePtr context, const String & default_database_name)
|
2011-10-30 11:30:52 +00:00
|
|
|
{
|
2020-08-10 02:58:08 +00:00
|
|
|
Poco::Logger * log = &Poco::Logger::get("loadMetadata");
|
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
String path = context->getPath() + "metadata";
|
2011-10-30 11:30:52 +00:00
|
|
|
|
2016-08-09 21:48:05 +00:00
|
|
|
/** There may exist 'force_restore_data' file, that means,
|
|
|
|
* skip safety threshold on difference of data parts while initializing tables.
|
|
|
|
* This file is deleted after successful loading of tables.
|
|
|
|
* (flag is "one-shot")
|
|
|
|
*/
|
2021-04-28 10:42:07 +00:00
|
|
|
auto force_restore_data_flag_file = fs::path(context->getFlagsPath()) / "force_restore_data";
|
|
|
|
bool has_force_restore_data_flag = fs::exists(force_restore_data_flag_file);
|
2016-08-09 21:48:05 +00:00
|
|
|
|
|
|
|
/// Loop over databases.
|
2017-06-15 20:08:26 +00:00
|
|
|
std::map<String, String> databases;
|
2021-04-28 10:42:07 +00:00
|
|
|
fs::directory_iterator dir_end;
|
|
|
|
for (fs::directory_iterator it(path); it != dir_end; ++it)
|
2011-10-30 11:30:52 +00:00
|
|
|
{
|
2021-04-28 10:42:07 +00:00
|
|
|
if (it->is_symlink())
|
2020-07-17 17:54:24 +00:00
|
|
|
continue;
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
if (it->is_directory())
|
|
|
|
continue;
|
|
|
|
|
2021-04-28 10:42:07 +00:00
|
|
|
const auto current_file = it->path().filename().string();
|
2022-06-23 19:38:43 +00:00
|
|
|
|
|
|
|
/// TODO: DETACH DATABASE PERMANENTLY ?
|
|
|
|
if (fs::path(current_file).extension() == ".sql")
|
2020-07-02 20:39:31 +00:00
|
|
|
{
|
2022-06-23 19:38:43 +00:00
|
|
|
String db_name = fs::path(current_file).stem();
|
|
|
|
if (!isSystemOrInformationSchema(db_name))
|
|
|
|
databases.emplace(unescapeForFileName(db_name), fs::path(path) / db_name);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Temporary fails may be left from previous server runs.
|
|
|
|
if (fs::path(current_file).extension() == ".tmp")
|
|
|
|
{
|
|
|
|
LOG_WARNING(log, "Removing temporary file {}", it->path().string());
|
|
|
|
try
|
2020-07-02 20:39:31 +00:00
|
|
|
{
|
2022-06-23 19:38:43 +00:00
|
|
|
fs::remove(it->path());
|
2020-07-02 20:39:31 +00:00
|
|
|
}
|
2022-06-23 19:38:43 +00:00
|
|
|
catch (...)
|
2020-08-10 02:58:08 +00:00
|
|
|
{
|
2022-06-23 19:38:43 +00:00
|
|
|
/// It does not prevent server to startup.
|
|
|
|
tryLogCurrentException(log);
|
2020-08-10 02:58:08 +00:00
|
|
|
}
|
2020-07-02 20:39:31 +00:00
|
|
|
}
|
2017-06-15 20:08:26 +00:00
|
|
|
}
|
|
|
|
|
2023-03-13 16:13:49 +00:00
|
|
|
checkIncompleteOrdinaryToAtomicConversion(context, databases);
|
|
|
|
|
2020-04-23 18:00:43 +00:00
|
|
|
/// clickhouse-local creates DatabaseMemory as default database by itself
|
|
|
|
/// For clickhouse-server we need create default database
|
|
|
|
bool create_default_db_if_not_exists = !default_database_name.empty();
|
2022-04-18 10:18:43 +00:00
|
|
|
bool metadata_dir_for_default_db_already_exists = databases.contains(default_database_name);
|
2020-04-23 18:00:43 +00:00
|
|
|
if (create_default_db_if_not_exists && !metadata_dir_for_default_db_already_exists)
|
2022-06-24 17:10:33 +00:00
|
|
|
{
|
|
|
|
checkUnsupportedVersion(context, default_database_name);
|
2021-10-18 06:06:38 +00:00
|
|
|
databases.emplace(default_database_name, std::filesystem::path(path) / escapeForFileName(default_database_name));
|
2022-06-24 17:10:33 +00:00
|
|
|
}
|
2019-11-28 19:40:51 +00:00
|
|
|
|
2021-08-26 13:19:52 +00:00
|
|
|
TablesLoader::Databases loaded_databases;
|
2019-08-03 11:02:40 +00:00
|
|
|
for (const auto & [name, db_path] : databases)
|
2021-08-26 13:19:52 +00:00
|
|
|
{
|
2019-08-03 11:02:40 +00:00
|
|
|
loadDatabase(context, name, db_path, has_force_restore_data_flag);
|
2021-09-01 19:42:49 +00:00
|
|
|
loaded_databases.insert({name, DatabaseCatalog::instance().getDatabase(name)});
|
2021-08-26 13:19:52 +00:00
|
|
|
}
|
|
|
|
|
2022-07-20 20:54:43 +00:00
|
|
|
auto mode = getLoadingStrictnessLevel(/* attach */ true, /* force_attach */ true, has_force_restore_data_flag);
|
|
|
|
TablesLoader loader{context, std::move(loaded_databases), mode};
|
2021-08-26 13:19:52 +00:00
|
|
|
loader.loadTables();
|
2021-09-01 19:42:49 +00:00
|
|
|
loader.startupTables();
|
2017-05-23 17:35:05 +00:00
|
|
|
|
2017-06-18 05:43:29 +00:00
|
|
|
if (has_force_restore_data_flag)
|
2018-12-10 18:56:37 +00:00
|
|
|
{
|
2018-12-09 15:11:46 +00:00
|
|
|
try
|
|
|
|
{
|
2021-04-28 10:42:07 +00:00
|
|
|
fs::remove(force_restore_data_flag_file);
|
2018-12-09 15:11:46 +00:00
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
2020-01-11 09:50:41 +00:00
|
|
|
tryLogCurrentException("Load metadata", "Can't remove force restore file to enable data sanity checks");
|
2018-12-09 15:11:46 +00:00
|
|
|
}
|
2018-12-10 18:56:37 +00:00
|
|
|
}
|
2017-06-18 05:43:29 +00:00
|
|
|
}
|
2017-05-23 17:35:05 +00:00
|
|
|
|
2021-09-09 09:37:51 +00:00
|
|
|
static void loadSystemDatabaseImpl(ContextMutablePtr context, const String & database_name, const String & default_engine)
|
2017-06-18 05:43:29 +00:00
|
|
|
{
|
2021-09-09 09:37:51 +00:00
|
|
|
String path = context->getPath() + "metadata/" + database_name;
|
2020-08-13 18:05:31 +00:00
|
|
|
String metadata_file = path + ".sql";
|
2023-08-02 21:36:54 +00:00
|
|
|
if (fs::exists(metadata_file + ".tmp"))
|
|
|
|
fs::remove(metadata_file + ".tmp");
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
if (fs::exists(fs::path(metadata_file)))
|
2017-06-15 20:08:26 +00:00
|
|
|
{
|
2017-06-18 05:43:29 +00:00
|
|
|
/// 'has_force_restore_data_flag' is true, to not fail on loading query_log table, if it is corrupted.
|
2021-09-09 09:37:51 +00:00
|
|
|
loadDatabase(context, database_name, path, true);
|
2017-06-15 20:08:26 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2022-06-24 17:10:33 +00:00
|
|
|
checkUnsupportedVersion(context, database_name);
|
2017-06-15 20:08:26 +00:00
|
|
|
/// Initialize system database manually
|
2020-08-12 20:40:13 +00:00
|
|
|
String database_create_query = "CREATE DATABASE ";
|
2021-09-09 09:37:51 +00:00
|
|
|
database_create_query += database_name;
|
|
|
|
database_create_query += " ENGINE=";
|
|
|
|
database_create_query += default_engine;
|
2022-07-20 20:54:43 +00:00
|
|
|
executeCreateQuery(database_create_query, context, database_name, "<no file>", true, true);
|
2017-06-15 20:08:26 +00:00
|
|
|
}
|
2021-09-09 09:37:51 +00:00
|
|
|
}
|
2017-06-15 20:08:26 +00:00
|
|
|
|
2022-08-08 14:06:08 +00:00
|
|
|
static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePtr context, const DatabasePtr & database,
|
|
|
|
const String & name, const String tmp_name)
|
2022-06-23 19:38:43 +00:00
|
|
|
{
|
2022-06-27 14:10:00 +00:00
|
|
|
/// It's kind of C++ script that creates temporary database with Atomic engine,
|
2022-06-23 19:38:43 +00:00
|
|
|
/// moves all tables to it, drops old database and then renames new one to old name.
|
|
|
|
|
|
|
|
String name_quoted = backQuoteIfNeed(name);
|
|
|
|
String tmp_name_quoted = backQuoteIfNeed(tmp_name);
|
|
|
|
|
|
|
|
LOG_INFO(log, "Will convert database {} from Ordinary to Atomic", name_quoted);
|
|
|
|
|
|
|
|
String create_database_query = fmt::format("CREATE DATABASE IF NOT EXISTS {}", tmp_name_quoted);
|
2023-11-09 15:03:51 +00:00
|
|
|
auto res = executeQuery(create_database_query, context, QueryFlags{ .internal = true }).second;
|
2022-06-23 19:38:43 +00:00
|
|
|
executeTrivialBlockIO(res, context);
|
|
|
|
res = {};
|
|
|
|
auto tmp_database = DatabaseCatalog::instance().getDatabase(tmp_name);
|
|
|
|
assert(tmp_database->getEngineName() == "Atomic");
|
|
|
|
|
|
|
|
size_t num_tables = 0;
|
2022-08-05 19:41:02 +00:00
|
|
|
std::unordered_set<String> inner_mv_tables;
|
2022-06-23 19:38:43 +00:00
|
|
|
for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
|
|
|
|
{
|
|
|
|
++num_tables;
|
|
|
|
auto id = iterator->table()->getStorageID();
|
|
|
|
id.database_name = tmp_name;
|
2022-08-05 19:41:02 +00:00
|
|
|
/// We need some uuid for checkTableCanBeRenamed
|
|
|
|
id.uuid = UUIDHelpers::generateV4();
|
2022-06-23 19:38:43 +00:00
|
|
|
iterator->table()->checkTableCanBeRenamed(id);
|
2022-08-05 19:41:02 +00:00
|
|
|
if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(iterator->table().get()))
|
|
|
|
{
|
|
|
|
/// We should not rename inner tables of MVs, because MVs are responsible for renaming it...
|
|
|
|
if (mv->hasInnerTable())
|
|
|
|
inner_mv_tables.emplace(mv->getTargetTable()->getStorageID().table_name);
|
|
|
|
}
|
2022-06-23 19:38:43 +00:00
|
|
|
}
|
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
LOG_INFO(log, "Will move {} tables to {} (including {} inner tables of MVs)", num_tables, tmp_name_quoted, inner_mv_tables.size());
|
2022-06-23 19:38:43 +00:00
|
|
|
|
|
|
|
for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next())
|
|
|
|
{
|
|
|
|
auto id = iterator->table()->getStorageID();
|
2022-08-05 19:41:02 +00:00
|
|
|
if (inner_mv_tables.contains(id.table_name))
|
|
|
|
{
|
|
|
|
LOG_DEBUG(log, "Do not rename {}, because it will be renamed together with MV", id.getNameForLogs());
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
String qualified_quoted_name = id.getFullTableName();
|
|
|
|
id.database_name = tmp_name;
|
|
|
|
String tmp_qualified_quoted_name = id.getFullTableName();
|
|
|
|
|
|
|
|
String move_table_query = fmt::format("RENAME TABLE {} TO {}", qualified_quoted_name, tmp_qualified_quoted_name);
|
2023-11-09 15:03:51 +00:00
|
|
|
res = executeQuery(move_table_query, context, QueryFlags{ .internal = true }).second;
|
2022-06-23 19:38:43 +00:00
|
|
|
executeTrivialBlockIO(res, context);
|
|
|
|
res = {};
|
|
|
|
}
|
|
|
|
|
|
|
|
LOG_INFO(log, "Moved all tables from {} to {}", name_quoted, tmp_name_quoted);
|
|
|
|
|
|
|
|
if (!database->empty())
|
|
|
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Database {} is not empty after moving tables", name_quoted);
|
|
|
|
|
|
|
|
String drop_query = fmt::format("DROP DATABASE {}", name_quoted);
|
2022-08-08 14:06:08 +00:00
|
|
|
context->setSetting("force_remove_data_recursively_on_drop", false);
|
2023-11-09 15:03:51 +00:00
|
|
|
res = executeQuery(drop_query, context, QueryFlags{ .internal = true }).second;
|
2022-06-23 19:38:43 +00:00
|
|
|
executeTrivialBlockIO(res, context);
|
|
|
|
res = {};
|
|
|
|
|
|
|
|
String rename_query = fmt::format("RENAME DATABASE {} TO {}", tmp_name_quoted, name_quoted);
|
2023-11-09 15:03:51 +00:00
|
|
|
res = executeQuery(rename_query, context, QueryFlags{ .internal = true }).second;
|
2022-06-23 19:38:43 +00:00
|
|
|
executeTrivialBlockIO(res, context);
|
|
|
|
|
|
|
|
LOG_INFO(log, "Finished database engine conversion of {}", name_quoted);
|
|
|
|
}
|
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
/// Converts database with Ordinary engine to Atomic. Does nothing if database is not Ordinary.
|
|
|
|
/// Can be called only during server startup when there are no queries from users.
|
|
|
|
static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const String & database_name, bool tables_started)
|
2022-06-23 19:38:43 +00:00
|
|
|
{
|
2022-08-05 19:41:02 +00:00
|
|
|
Poco::Logger * log = &Poco::Logger::get("loadMetadata");
|
|
|
|
|
|
|
|
auto database = DatabaseCatalog::instance().getDatabase(database_name);
|
|
|
|
if (!database)
|
|
|
|
{
|
|
|
|
LOG_WARNING(log, "Database {} not found (while trying to convert it from Ordinary to Atomic)", database_name);
|
2022-06-23 19:38:43 +00:00
|
|
|
return;
|
2022-08-05 19:41:02 +00:00
|
|
|
}
|
2022-06-23 19:38:43 +00:00
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
if (database->getEngineName() != "Ordinary")
|
2022-06-23 19:38:43 +00:00
|
|
|
return;
|
|
|
|
|
2022-08-08 14:06:08 +00:00
|
|
|
Strings permanently_detached_tables = database->getNamesOfPermanentlyDetachedTables();
|
|
|
|
if (!permanently_detached_tables.empty())
|
|
|
|
{
|
|
|
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot automatically convert database {} from Ordinary to Atomic, "
|
|
|
|
"because it contains permanently detached tables ({}) that were not loaded during startup. "
|
|
|
|
"Attach these tables, so server will load and convert them",
|
|
|
|
database_name, fmt::join(permanently_detached_tables, ", "));
|
|
|
|
}
|
|
|
|
|
2023-03-13 16:13:49 +00:00
|
|
|
String tmp_name = fmt::format(ORDINARY_TO_ATOMIC_PREFIX"{}.{}", database_name, thread_local_rng());
|
2022-08-08 14:06:08 +00:00
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
try
|
|
|
|
{
|
2022-08-05 19:41:02 +00:00
|
|
|
if (!tables_started)
|
|
|
|
{
|
|
|
|
/// It's not quite correct to run DDL queries while database is not started up.
|
2023-11-18 18:07:59 +00:00
|
|
|
ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive, CurrentMetrics::StartupSystemTablesThreadsScheduled);
|
2022-08-17 10:17:37 +00:00
|
|
|
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE);
|
2022-08-05 19:41:02 +00:00
|
|
|
}
|
2022-06-23 19:38:43 +00:00
|
|
|
|
|
|
|
auto local_context = Context::createCopy(context);
|
2022-09-09 16:27:19 +00:00
|
|
|
|
|
|
|
/// We have to stop background operations that may lock table for share to avoid DEADLOCK_AVOIDED error
|
|
|
|
/// on moving tables from Ordinary database. Server has not started to accept connections yet,
|
|
|
|
/// so there are no user queries, only background operations
|
|
|
|
LOG_INFO(log, "Will stop background operations to be able to rename tables in Ordinary database {}", database_name);
|
2022-09-12 11:53:43 +00:00
|
|
|
static const auto actions_to_stop = {
|
|
|
|
ActionLocks::PartsMerge, ActionLocks::PartsFetch, ActionLocks::PartsSend, ActionLocks::DistributedSend
|
|
|
|
};
|
|
|
|
for (const auto & action : actions_to_stop)
|
2022-09-09 16:27:19 +00:00
|
|
|
InterpreterSystemQuery::startStopActionInDatabase(action, /* start */ false, database_name, database, context, log);
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
local_context->setSetting("check_table_dependencies", false);
|
2023-02-01 23:30:49 +00:00
|
|
|
local_context->setSetting("check_referential_table_dependencies", false);
|
2022-08-08 14:06:08 +00:00
|
|
|
convertOrdinaryDatabaseToAtomic(log, local_context, database, database_name, tmp_name);
|
2022-06-23 19:38:43 +00:00
|
|
|
|
2022-09-09 16:27:19 +00:00
|
|
|
LOG_INFO(log, "Will start background operations after renaming tables in database {}", database_name);
|
2022-09-12 11:53:43 +00:00
|
|
|
for (const auto & action : actions_to_stop)
|
2022-09-09 16:27:19 +00:00
|
|
|
InterpreterSystemQuery::startStopActionInDatabase(action, /* start */ true, database_name, database, context, log);
|
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
auto new_database = DatabaseCatalog::instance().getDatabase(database_name);
|
2022-06-27 22:26:52 +00:00
|
|
|
UUID db_uuid = new_database->getUUID();
|
|
|
|
std::vector<UUID> tables_uuids;
|
|
|
|
for (auto iterator = new_database->getTablesIterator(context); iterator->isValid(); iterator->next())
|
|
|
|
tables_uuids.push_back(iterator->uuid());
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
/// Reload database just in case (and update logger name)
|
2022-08-05 19:41:02 +00:00
|
|
|
String detach_query = fmt::format("DETACH DATABASE {}", backQuoteIfNeed(database_name));
|
2023-11-09 15:03:51 +00:00
|
|
|
auto res = executeQuery(detach_query, context, QueryFlags{ .internal = true }).second;
|
2022-06-23 19:38:43 +00:00
|
|
|
executeTrivialBlockIO(res, context);
|
|
|
|
res = {};
|
|
|
|
|
2022-06-27 22:26:52 +00:00
|
|
|
/// Unlock UUID mapping, because it will be locked again on database reload.
|
|
|
|
/// It's safe to do during metadata loading, because cleanup task is not started yet.
|
|
|
|
DatabaseCatalog::instance().removeUUIDMappingFinally(db_uuid);
|
|
|
|
for (const auto & uuid : tables_uuids)
|
|
|
|
DatabaseCatalog::instance().removeUUIDMappingFinally(uuid);
|
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
String path = context->getPath() + "metadata/" + escapeForFileName(database_name);
|
|
|
|
/// force_restore_data is needed to re-create metadata symlinks
|
|
|
|
loadDatabase(context, database_name, path, /* force_restore_data */ true);
|
|
|
|
|
2022-06-23 19:38:43 +00:00
|
|
|
TablesLoader::Databases databases =
|
|
|
|
{
|
2022-08-05 19:41:02 +00:00
|
|
|
{database_name, DatabaseCatalog::instance().getDatabase(database_name)},
|
2022-06-23 19:38:43 +00:00
|
|
|
};
|
2022-07-20 20:54:43 +00:00
|
|
|
TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
|
2022-06-23 19:38:43 +00:00
|
|
|
loader.loadTables();
|
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
/// Startup tables if they were started before conversion and detach/attach
|
|
|
|
if (tables_started)
|
|
|
|
loader.startupTables();
|
2022-06-23 19:38:43 +00:00
|
|
|
}
|
|
|
|
catch (Exception & e)
|
|
|
|
{
|
2022-08-08 14:06:08 +00:00
|
|
|
e.addMessage("Exception while trying to convert database {} from Ordinary to Atomic. It may be in some intermediate state."
|
2022-09-21 05:10:33 +00:00
|
|
|
" You can finish conversion manually by moving the rest tables from {} to {} (using RENAME TABLE)"
|
|
|
|
" and executing DROP DATABASE {} and RENAME DATABASE {} TO {}.",
|
2022-08-08 14:06:08 +00:00
|
|
|
database_name, database_name, tmp_name, database_name, tmp_name, database_name);
|
2022-06-23 19:38:43 +00:00
|
|
|
throw;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-05 19:41:02 +00:00
|
|
|
void maybeConvertSystemDatabase(ContextMutablePtr context)
|
|
|
|
{
|
|
|
|
/// TODO remove this check, convert system database unconditionally
|
|
|
|
if (context->getSettingsRef().allow_deprecated_database_ordinary)
|
|
|
|
return;
|
|
|
|
|
|
|
|
maybeConvertOrdinaryDatabaseToAtomic(context, DatabaseCatalog::SYSTEM_DATABASE, /* tables_started */ false);
|
|
|
|
}
|
|
|
|
|
|
|
|
void convertDatabasesEnginesIfNeed(ContextMutablePtr context)
|
|
|
|
{
|
|
|
|
auto convert_flag_path = fs::path(context->getFlagsPath()) / "convert_ordinary_to_atomic";
|
|
|
|
if (!fs::exists(convert_flag_path))
|
|
|
|
return;
|
|
|
|
|
|
|
|
LOG_INFO(&Poco::Logger::get("loadMetadata"), "Found convert_ordinary_to_atomic file in flags directory, "
|
|
|
|
"will try to convert all Ordinary databases to Atomic");
|
|
|
|
|
|
|
|
for (const auto & [name, _] : DatabaseCatalog::instance().getDatabases())
|
|
|
|
if (name != DatabaseCatalog::SYSTEM_DATABASE)
|
|
|
|
maybeConvertOrdinaryDatabaseToAtomic(context, name, /* tables_started */ true);
|
2023-03-13 16:13:49 +00:00
|
|
|
|
|
|
|
LOG_INFO(&Poco::Logger::get("loadMetadata"), "Conversion finished, removing convert_ordinary_to_atomic flag");
|
|
|
|
fs::remove(convert_flag_path);
|
2022-08-05 19:41:02 +00:00
|
|
|
}
|
2021-09-01 19:42:49 +00:00
|
|
|
|
|
|
|
void startupSystemTables()
|
|
|
|
{
|
2023-11-18 18:07:59 +00:00
|
|
|
ThreadPool pool(CurrentMetrics::StartupSystemTablesThreads, CurrentMetrics::StartupSystemTablesThreadsActive, CurrentMetrics::StartupSystemTablesThreadsScheduled);
|
2022-07-20 20:54:43 +00:00
|
|
|
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE);
|
2021-09-10 10:11:40 +00:00
|
|
|
}
|
|
|
|
|
2021-09-09 09:37:51 +00:00
|
|
|
void loadMetadataSystem(ContextMutablePtr context)
|
|
|
|
{
|
|
|
|
loadSystemDatabaseImpl(context, DatabaseCatalog::SYSTEM_DATABASE, "Atomic");
|
|
|
|
loadSystemDatabaseImpl(context, DatabaseCatalog::INFORMATION_SCHEMA, "Memory");
|
|
|
|
loadSystemDatabaseImpl(context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, "Memory");
|
2021-09-10 10:11:40 +00:00
|
|
|
|
|
|
|
TablesLoader::Databases databases =
|
|
|
|
{
|
|
|
|
{DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::instance().getSystemDatabase()},
|
|
|
|
{DatabaseCatalog::INFORMATION_SCHEMA, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA)},
|
|
|
|
{DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)},
|
|
|
|
};
|
2022-07-20 20:54:43 +00:00
|
|
|
TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
|
2021-09-10 10:11:40 +00:00
|
|
|
loader.loadTables();
|
|
|
|
/// Will startup tables in system database after all databases are loaded.
|
2011-10-30 11:30:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|