This commit is contained in:
Alexander Tokmakov 2022-07-20 22:54:43 +02:00
parent e295fd5090
commit 6e77155dc8
25 changed files with 143 additions and 89 deletions

View File

@ -416,9 +416,9 @@ UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const
return UUIDHelpers::Nil;
}
void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool force_restore, bool /*force_attach*/)
void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, LoadingStrictnessLevel mode)
{
if (!force_restore)
if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
return;
/// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken
@ -435,17 +435,17 @@ void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool f
}
void DatabaseAtomic::loadStoredObjects(
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables)
ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
{
beforeLoadingMetadata(local_context, force_restore, force_attach);
DatabaseOrdinary::loadStoredObjects(local_context, force_restore, force_attach, skip_startup_tables);
beforeLoadingMetadata(local_context, mode);
DatabaseOrdinary::loadStoredObjects(local_context, mode, skip_startup_tables);
}
void DatabaseAtomic::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
void DatabaseAtomic::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{
DatabaseOrdinary::startupTables(thread_pool, force_restore, force_attach);
DatabaseOrdinary::startupTables(thread_pool, mode);
if (!force_restore)
if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
return;
NameToPathMap table_names;

View File

@ -47,11 +47,11 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
void beforeLoadingMetadata(ContextMutablePtr context, bool force_restore, bool force_attach) override;
void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
/// Atomic database cannot be detached if there is detached table which still in use
void assertCanBeDetached(bool cleanup) override;

View File

@ -38,7 +38,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_,
void DatabaseLazy::loadStoredObjects(
ContextMutablePtr local_context, bool /* force_restore */, bool /*force_attach*/, bool /* skip_startup_tables */)
ContextMutablePtr local_context, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
{
iterateMetadataFiles(local_context, [this, &local_context](const String & file_name)
{

View File

@ -26,7 +26,7 @@ public:
bool canContainDistributedTables() const override { return false; }
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
void createTable(
ContextPtr context,

View File

@ -81,7 +81,7 @@ DatabaseOrdinary::DatabaseOrdinary(
}
void DatabaseOrdinary::loadStoredObjects(
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables)
ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
{
/** Tables load faster if they are loaded in sorted (by name) order.
* Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
@ -89,6 +89,7 @@ void DatabaseOrdinary::loadStoredObjects(
*/
ParsedTablesMetadata metadata;
bool force_attach = LoadingStrictnessLevel::FORCE_ATTACH <= mode;
loadTablesMetadata(local_context, metadata, force_attach);
size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries;
@ -118,7 +119,7 @@ void DatabaseOrdinary::loadStoredObjects(
{
pool.scheduleOrThrowOnError([&]()
{
loadTableFromMetadata(local_context, path, name, ast, force_restore);
loadTableFromMetadata(local_context, path, name, ast, mode);
/// Messages, so that it's not boring to wait for the server to load for a long time.
logAboutProgress(log, ++dictionaries_processed, metadata.total_dictionaries, watch);
@ -140,7 +141,7 @@ void DatabaseOrdinary::loadStoredObjects(
{
pool.scheduleOrThrowOnError([&]()
{
loadTableFromMetadata(local_context, path, name, ast, force_restore);
loadTableFromMetadata(local_context, path, name, ast, mode);
/// Messages, so that it's not boring to wait for the server to load for a long time.
logAboutProgress(log, ++tables_processed, total_tables, watch);
@ -153,7 +154,7 @@ void DatabaseOrdinary::loadStoredObjects(
if (!skip_startup_tables)
{
/// After all tables was basically initialized, startup them.
startupTables(pool, force_restore, force_attach);
startupTables(pool, mode);
}
}
@ -238,7 +239,8 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
TSA_SUPPRESS_WARNING_FOR_READ(database_name), tables_in_database, dictionaries_in_database);
}
void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore)
void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast,
LoadingStrictnessLevel mode)
{
assert(name.database == TSA_SUPPRESS_WARNING_FOR_READ(database_name));
const auto & create_query = ast->as<const ASTCreateQuery &>();
@ -248,11 +250,10 @@ void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, co
create_query,
*this,
name.database,
file_path,
force_restore);
file_path, LoadingStrictnessLevel::FORCE_RESTORE <= mode);
}
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_restore*/, bool /*force_attach*/)
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel /*mode*/)
{
LOG_INFO(log, "Starting up tables.");

View File

@ -21,15 +21,16 @@ public:
String getEngineName() const override { return "Ordinary"; }
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
bool supportsLoadingInTopologicalOrder() const override { return true; }
void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override;
void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) override;
void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast,
LoadingStrictnessLevel mode) override;
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
void alterTable(
ContextPtr context,

View File

@ -232,7 +232,7 @@ void DatabaseReplicated::fillClusterAuthInfo(String collection_name, const Poco:
cluster_auth_info.cluster_secure_connection = config_ref.getBool(config_prefix + ".cluster_secure_connection", false);
}
void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach, bool is_create_query)
void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel mode)
{
try
{
@ -250,6 +250,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach,
}
replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName();
bool is_create_query = mode == LoadingStrictnessLevel::CREATE;
String replica_host_id;
if (current_zookeeper->tryGet(replica_path, replica_host_id))
@ -290,7 +291,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach,
}
catch (...)
{
if (!force_attach)
if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
throw;
/// It's server startup, ignore error.
@ -339,6 +340,8 @@ void DatabaseReplicated::createEmptyLogEntry(const ZooKeeperPtr & current_zookee
bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms)
{
if (!ddl_worker)
return false;
return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms);
}
@ -374,21 +377,21 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt
createEmptyLogEntry(current_zookeeper);
}
void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool /*force_restore*/, bool force_attach)
void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, LoadingStrictnessLevel mode)
{
tryConnectToZooKeeperAndInitDatabase(force_attach, /* is_create_query */ !force_attach);
tryConnectToZooKeeperAndInitDatabase(mode);
}
void DatabaseReplicated::loadStoredObjects(
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables)
ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
{
beforeLoadingMetadata(local_context, force_restore, force_attach);
DatabaseAtomic::loadStoredObjects(local_context, force_restore, force_attach, skip_startup_tables);
beforeLoadingMetadata(local_context, mode);
DatabaseAtomic::loadStoredObjects(local_context, mode, skip_startup_tables);
}
void DatabaseReplicated::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
void DatabaseReplicated::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach);
DatabaseAtomic::startupTables(thread_pool, mode);
ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
if (is_probably_dropped)
return;

View File

@ -64,11 +64,11 @@ public:
void drop(ContextPtr /*context*/) override;
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
void beforeLoadingMetadata(ContextMutablePtr context, bool force_restore, bool force_attach) override;
void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
void shutdown() override;
@ -78,7 +78,7 @@ public:
friend struct DatabaseReplicatedTask;
friend class DatabaseReplicatedDDLWorker;
private:
void tryConnectToZooKeeperAndInitDatabase(bool force_attach, bool is_create_query);
void tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel mode);
bool createDatabaseNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);
void createReplicaNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);

View File

@ -35,7 +35,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
chassert(!database->is_probably_dropped);
auto zookeeper = getAndSetZooKeeper();
if (database->is_readonly)
database->tryConnectToZooKeeperAndInitDatabase(/* force_attach */ false, /* is_create_query */ false);
database->tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel::ATTACH);
initializeReplication();
initialized = true;
return true;

View File

@ -1,12 +1,13 @@
#pragma once
#include <base/types.h>
#include <Core/UUID.h>
#include <Databases/LoadingStrictnessLevel.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/IAST_fwd.h>
#include <Storages/IStorage_fwd.h>
#include <Interpreters/Context_fwd.h>
#include <base/types.h>
#include <Common/Exception.h>
#include <Common/ThreadPool.h>
#include <Core/UUID.h>
#include <ctime>
#include <functional>
@ -132,18 +133,15 @@ public:
/// You can call only once, right after the object is created.
virtual void loadStoredObjects( /// NOLINT
ContextMutablePtr /*context*/,
bool /*force_restore*/,
bool /*force_attach*/ = false,
bool /* skip_startup_tables */ = false)
LoadingStrictnessLevel /*mode*/,
bool /* skip_startup_tables */)
{
}
virtual bool supportsLoadingInTopologicalOrder() const { return false; }
virtual void beforeLoadingMetadata(
ContextMutablePtr /*context*/,
bool /*force_restore*/,
bool /*force_attach*/)
ContextMutablePtr /*context*/, LoadingStrictnessLevel /*mode*/)
{
}
@ -152,12 +150,13 @@ public:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented");
}
virtual void loadTableFromMetadata(ContextMutablePtr /*local_context*/, const String & /*file_path*/, const QualifiedTableName & /*name*/, const ASTPtr & /*ast*/, bool /*force_restore*/)
virtual void loadTableFromMetadata(ContextMutablePtr /*local_context*/, const String & /*file_path*/, const QualifiedTableName & /*name*/, const ASTPtr & /*ast*/,
LoadingStrictnessLevel /*mode*/)
{
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented");
}
virtual void startupTables(ThreadPool & /*thread_pool*/, bool /*force_restore*/, bool /*force_attach*/) {}
virtual void startupTables(ThreadPool & /*thread_pool*/, LoadingStrictnessLevel /*mode*/) {}
/// Check the existence of the table in memory (attached).
virtual bool isTableExist(const String & name, ContextPtr context) const = 0;

View File

@ -0,0 +1,27 @@
#include <Databases/LoadingStrictnessLevel.h>
namespace DB
{
LoadingStrictnessLevel getLoadingStrictnessLevel(bool attach, bool force_attach, bool force_restore)
{
if (force_restore)
{
assert(attach);
assert(force_attach);
return LoadingStrictnessLevel::FORCE_RESTORE;
}
if (force_attach)
{
assert(attach);
return LoadingStrictnessLevel::FORCE_ATTACH;
}
if (attach)
return LoadingStrictnessLevel::ATTACH;
return LoadingStrictnessLevel::CREATE;
}
}

View File

@ -0,0 +1,16 @@
#pragma once
namespace DB
{
enum class LoadingStrictnessLevel
{
CREATE = 0,
ATTACH = 1,
FORCE_ATTACH = 2,
FORCE_RESTORE = 3,
};
LoadingStrictnessLevel getLoadingStrictnessLevel(bool attach, bool force_attach, bool force_restore);
}

View File

@ -63,11 +63,11 @@ void DatabaseMaterializedMySQL::setException(const std::exception_ptr & exceptio
exception = exception_;
}
void DatabaseMaterializedMySQL::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
void DatabaseMaterializedMySQL::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach);
DatabaseAtomic::startupTables(thread_pool, mode);
if (!force_attach)
if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
materialize_thread.assertMySQLAvailable();
materialize_thread.startSynchronization();

View File

@ -48,7 +48,7 @@ protected:
public:
String getEngineName() const override { return "MaterializedMySQL"; }
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override;

View File

@ -398,7 +398,7 @@ String DatabaseMySQL::getMetadataPath() const
return metadata_path;
}
void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /*force_attach*/, bool /* skip_startup_tables */)
void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
{
std::lock_guard<std::mutex> lock{mutex};

View File

@ -76,7 +76,7 @@ public:
void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override;
void loadStoredObjects(ContextMutablePtr, bool, bool force_attach, bool skip_startup_tables) override;
void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
StoragePtr detachTable(ContextPtr context, const String & table_name) override;

View File

@ -125,9 +125,9 @@ void DatabaseMaterializedPostgreSQL::startSynchronization()
}
void DatabaseMaterializedPostgreSQL::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
void DatabaseMaterializedPostgreSQL::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach);
DatabaseAtomic::startupTables(thread_pool, mode);
startup_task->activateAndSchedule();
}

View File

@ -40,7 +40,7 @@ public:
String getMetadataPath() const override { return metadata_path; }
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
DatabaseTablesIteratorPtr
getTablesIterator(ContextPtr context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override;

View File

@ -290,7 +290,7 @@ void DatabasePostgreSQL::drop(ContextPtr /*context*/)
}
void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool, bool /*force_attach*/, bool /* skip_startup_tables */)
void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
{
{
std::lock_guard<std::mutex> lock{mutex};

View File

@ -45,7 +45,7 @@ public:
bool empty() const override;
void loadStoredObjects(ContextMutablePtr, bool, bool force_attach, bool skip_startup_tables) override;
void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;

View File

@ -62,11 +62,10 @@ void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, Atomic
}
}
TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_, bool force_attach_)
TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_)
: global_context(global_context_)
, databases(std::move(databases_))
, force_restore(force_restore_)
, force_attach(force_attach_)
, strictness_mode(strictness_mode_)
{
metadata.default_database = global_context->getCurrentDatabase();
log = &Poco::Logger::get("TablesLoader");
@ -83,7 +82,7 @@ void TablesLoader::loadTables()
if (need_resolve_dependencies && database.second->supportsLoadingInTopologicalOrder())
databases_to_load.push_back(database.first);
else
database.second->loadStoredObjects(global_context, force_restore, force_attach, true);
database.second->loadStoredObjects(global_context, strictness_mode, /* skip_startup_tables */ true);
}
if (databases_to_load.empty())
@ -92,8 +91,9 @@ void TablesLoader::loadTables()
/// Read and parse metadata from Ordinary, Atomic, Materialized*, Replicated, etc databases. Build dependency graph.
for (auto & database_name : databases_to_load)
{
databases[database_name]->beforeLoadingMetadata(global_context, force_restore, force_attach);
databases[database_name]->loadTablesMetadata(global_context, metadata, force_attach);
databases[database_name]->beforeLoadingMetadata(global_context, strictness_mode);
bool is_startup = LoadingStrictnessLevel::FORCE_ATTACH <= strictness_mode;
databases[database_name]->loadTablesMetadata(global_context, metadata, is_startup);
}
LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec",
@ -119,7 +119,7 @@ void TablesLoader::startupTables()
{
/// Startup tables after all tables are loaded. Background tasks (merges, mutations, etc) may slow down data parts loading.
for (auto & database : databases)
database.second->startupTables(pool, force_restore, force_attach);
database.second->startupTables(pool, strictness_mode);
}
@ -253,7 +253,7 @@ void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level
pool.scheduleOrThrowOnError([this, load_context, total_tables, &table_name]()
{
const auto & path_and_query = metadata.parsed_tables[table_name];
databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, force_restore);
databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, strictness_mode);
logAboutProgress(log, ++tables_processed, total_tables, stopwatch);
});
}

View File

@ -1,14 +1,15 @@
#pragma once
#include <Core/Types.h>
#include <Core/QualifiedTableName.h>
#include <Parsers/IAST_fwd.h>
#include <Interpreters/Context_fwd.h>
#include <Common/ThreadPool.h>
#include <Common/Stopwatch.h>
#include <map>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
#include <mutex>
#include <Core/QualifiedTableName.h>
#include <Core/Types.h>
#include <Databases/LoadingStrictnessLevel.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/IAST_fwd.h>
#include <Common/Stopwatch.h>
#include <Common/ThreadPool.h>
namespace Poco
{
@ -78,7 +79,7 @@ class TablesLoader
public:
using Databases = std::map<String, DatabasePtr>;
TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_ = false, bool force_attach_ = false);
TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_);
TablesLoader() = delete;
void loadTables();
@ -87,8 +88,7 @@ public:
private:
ContextMutablePtr global_context;
Databases databases;
bool force_restore;
bool force_attach;
LoadingStrictnessLevel strictness_mode;
Strings databases_to_load;
ParsedTablesMetadata metadata;

View File

@ -249,6 +249,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path);
bool need_lock_uuid = internal || need_write_metadata;
auto mode = getLoadingStrictnessLevel(create.attach, force_attach, has_force_restore_data_flag);
/// Lock uuid, so we will known it's already in use.
/// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach);
@ -303,8 +304,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
if (!load_database_without_tables)
{
/// We use global context here, because storages lifetime is bigger than query context lifetime
TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, has_force_restore_data_flag, create.attach && force_attach}; //-V560
TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode}; //-V560
loader.loadTables();
loader.startupTables();
}

View File

@ -760,6 +760,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
{
const auto database_name = query.getDatabase();
auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, "");
auto database = DatabaseCatalog::instance().getDatabase(database_name);
if (auto * ptr = typeid_cast<DatabaseReplicated *>(database.get()))
@ -767,8 +768,7 @@ void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
LOG_TRACE(log, "Synchronizing entries in the database replica's (name: {}) queue with the log", database_name);
if (!ptr->waitForReplicaToProcessAllEntries(getContext()->getSettingsRef().receive_timeout.totalMilliseconds()))
{
LOG_ERROR(log, "SYNC DATABASE REPLICA {}: Timed out!", database_name);
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: command timed out. " \
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: database is readonly or command timed out. " \
"See the 'receive_timeout' setting", database_name);
}
LOG_TRACE(log, "SYNC DATABASE REPLICA {}: OK", database_name);

View File

@ -38,6 +38,7 @@ static void executeCreateQuery(
ContextMutablePtr context,
const String & database,
const String & file_name,
bool create,
bool has_force_restore_data_flag)
{
ParserCreateQuery parser;
@ -49,8 +50,11 @@ static void executeCreateQuery(
InterpreterCreateQuery interpreter(ast, context);
interpreter.setInternal(true);
if (!create)
{
interpreter.setForceAttach(true);
interpreter.setForceRestoreData(has_force_restore_data_flag);
}
interpreter.setLoadDatabaseWithoutTables(true);
interpreter.execute();
}
@ -86,7 +90,7 @@ static void loadDatabase(
try
{
executeCreateQuery(database_attach_query, context, database, database_metadata_file, force_restore_data);
executeCreateQuery(database_attach_query, context, database, database_metadata_file, /* create */ true, force_restore_data);
}
catch (Exception & e)
{
@ -173,7 +177,8 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
loaded_databases.insert({name, DatabaseCatalog::instance().getDatabase(name)});
}
TablesLoader loader{context, std::move(loaded_databases), has_force_restore_data_flag, /* force_attach */ true};
auto mode = getLoadingStrictnessLevel(/* attach */ true, /* force_attach */ true, has_force_restore_data_flag);
TablesLoader loader{context, std::move(loaded_databases), mode};
loader.loadTables();
loader.startupTables();
@ -207,7 +212,7 @@ static void loadSystemDatabaseImpl(ContextMutablePtr context, const String & dat
database_create_query += database_name;
database_create_query += " ENGINE=";
database_create_query += default_engine;
executeCreateQuery(database_create_query, context, database_name, "<no file>", true);
executeCreateQuery(database_create_query, context, database_name, "<no file>", true, true);
}
}
@ -315,7 +320,7 @@ void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const Datab
{
{DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::instance().getSystemDatabase()},
};
TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true};
TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
loader.loadTables();
/// Will startup tables usual way
@ -331,7 +336,7 @@ void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const Datab
void startupSystemTables()
{
ThreadPool pool;
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, /* force_restore */ true, /* force_attach */ true);
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE);
}
void loadMetadataSystem(ContextMutablePtr context)
@ -346,7 +351,7 @@ void loadMetadataSystem(ContextMutablePtr context)
{DatabaseCatalog::INFORMATION_SCHEMA, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA)},
{DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)},
};
TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true};
TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
loader.loadTables();
/// Will startup tables in system database after all databases are loaded.
}