This commit is contained in:
Alexander Tokmakov 2022-07-20 22:54:43 +02:00
parent e295fd5090
commit 6e77155dc8
25 changed files with 143 additions and 89 deletions

View File

@ -416,9 +416,9 @@ UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const
return UUIDHelpers::Nil; return UUIDHelpers::Nil;
} }
void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool force_restore, bool /*force_attach*/) void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, LoadingStrictnessLevel mode)
{ {
if (!force_restore) if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
return; return;
/// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken /// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken
@ -435,17 +435,17 @@ void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool f
} }
void DatabaseAtomic::loadStoredObjects( void DatabaseAtomic::loadStoredObjects(
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables) ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
{ {
beforeLoadingMetadata(local_context, force_restore, force_attach); beforeLoadingMetadata(local_context, mode);
DatabaseOrdinary::loadStoredObjects(local_context, force_restore, force_attach, skip_startup_tables); DatabaseOrdinary::loadStoredObjects(local_context, mode, skip_startup_tables);
} }
void DatabaseAtomic::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) void DatabaseAtomic::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{ {
DatabaseOrdinary::startupTables(thread_pool, force_restore, force_attach); DatabaseOrdinary::startupTables(thread_pool, mode);
if (!force_restore) if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
return; return;
NameToPathMap table_names; NameToPathMap table_names;

View File

@ -47,11 +47,11 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override; void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
void beforeLoadingMetadata(ContextMutablePtr context, bool force_restore, bool force_attach) override; void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override; void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
/// Atomic database cannot be detached if there is detached table which still in use /// Atomic database cannot be detached if there is detached table which still in use
void assertCanBeDetached(bool cleanup) override; void assertCanBeDetached(bool cleanup) override;

View File

@ -38,7 +38,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_,
void DatabaseLazy::loadStoredObjects( void DatabaseLazy::loadStoredObjects(
ContextMutablePtr local_context, bool /* force_restore */, bool /*force_attach*/, bool /* skip_startup_tables */) ContextMutablePtr local_context, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
{ {
iterateMetadataFiles(local_context, [this, &local_context](const String & file_name) iterateMetadataFiles(local_context, [this, &local_context](const String & file_name)
{ {

View File

@ -26,7 +26,7 @@ public:
bool canContainDistributedTables() const override { return false; } bool canContainDistributedTables() const override { return false; }
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override; void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
void createTable( void createTable(
ContextPtr context, ContextPtr context,

View File

@ -81,7 +81,7 @@ DatabaseOrdinary::DatabaseOrdinary(
} }
void DatabaseOrdinary::loadStoredObjects( void DatabaseOrdinary::loadStoredObjects(
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables) ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
{ {
/** Tables load faster if they are loaded in sorted (by name) order. /** Tables load faster if they are loaded in sorted (by name) order.
* Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order, * Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
@ -89,6 +89,7 @@ void DatabaseOrdinary::loadStoredObjects(
*/ */
ParsedTablesMetadata metadata; ParsedTablesMetadata metadata;
bool force_attach = LoadingStrictnessLevel::FORCE_ATTACH <= mode;
loadTablesMetadata(local_context, metadata, force_attach); loadTablesMetadata(local_context, metadata, force_attach);
size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries; size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries;
@ -118,7 +119,7 @@ void DatabaseOrdinary::loadStoredObjects(
{ {
pool.scheduleOrThrowOnError([&]() pool.scheduleOrThrowOnError([&]()
{ {
loadTableFromMetadata(local_context, path, name, ast, force_restore); loadTableFromMetadata(local_context, path, name, ast, mode);
/// Messages, so that it's not boring to wait for the server to load for a long time. /// Messages, so that it's not boring to wait for the server to load for a long time.
logAboutProgress(log, ++dictionaries_processed, metadata.total_dictionaries, watch); logAboutProgress(log, ++dictionaries_processed, metadata.total_dictionaries, watch);
@ -140,7 +141,7 @@ void DatabaseOrdinary::loadStoredObjects(
{ {
pool.scheduleOrThrowOnError([&]() pool.scheduleOrThrowOnError([&]()
{ {
loadTableFromMetadata(local_context, path, name, ast, force_restore); loadTableFromMetadata(local_context, path, name, ast, mode);
/// Messages, so that it's not boring to wait for the server to load for a long time. /// Messages, so that it's not boring to wait for the server to load for a long time.
logAboutProgress(log, ++tables_processed, total_tables, watch); logAboutProgress(log, ++tables_processed, total_tables, watch);
@ -153,7 +154,7 @@ void DatabaseOrdinary::loadStoredObjects(
if (!skip_startup_tables) if (!skip_startup_tables)
{ {
/// After all tables was basically initialized, startup them. /// After all tables was basically initialized, startup them.
startupTables(pool, force_restore, force_attach); startupTables(pool, mode);
} }
} }
@ -238,7 +239,8 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
TSA_SUPPRESS_WARNING_FOR_READ(database_name), tables_in_database, dictionaries_in_database); TSA_SUPPRESS_WARNING_FOR_READ(database_name), tables_in_database, dictionaries_in_database);
} }
void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast,
LoadingStrictnessLevel mode)
{ {
assert(name.database == TSA_SUPPRESS_WARNING_FOR_READ(database_name)); assert(name.database == TSA_SUPPRESS_WARNING_FOR_READ(database_name));
const auto & create_query = ast->as<const ASTCreateQuery &>(); const auto & create_query = ast->as<const ASTCreateQuery &>();
@ -248,11 +250,10 @@ void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, co
create_query, create_query,
*this, *this,
name.database, name.database,
file_path, file_path, LoadingStrictnessLevel::FORCE_RESTORE <= mode);
force_restore);
} }
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_restore*/, bool /*force_attach*/) void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel /*mode*/)
{ {
LOG_INFO(log, "Starting up tables."); LOG_INFO(log, "Starting up tables.");

View File

@ -21,15 +21,16 @@ public:
String getEngineName() const override { return "Ordinary"; } String getEngineName() const override { return "Ordinary"; }
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override; void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
bool supportsLoadingInTopologicalOrder() const override { return true; } bool supportsLoadingInTopologicalOrder() const override { return true; }
void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override; void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override;
void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) override; void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast,
LoadingStrictnessLevel mode) override;
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override; void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
void alterTable( void alterTable(
ContextPtr context, ContextPtr context,

View File

@ -232,7 +232,7 @@ void DatabaseReplicated::fillClusterAuthInfo(String collection_name, const Poco:
cluster_auth_info.cluster_secure_connection = config_ref.getBool(config_prefix + ".cluster_secure_connection", false); cluster_auth_info.cluster_secure_connection = config_ref.getBool(config_prefix + ".cluster_secure_connection", false);
} }
void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach, bool is_create_query) void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel mode)
{ {
try try
{ {
@ -250,6 +250,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach,
} }
replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName(); replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName();
bool is_create_query = mode == LoadingStrictnessLevel::CREATE;
String replica_host_id; String replica_host_id;
if (current_zookeeper->tryGet(replica_path, replica_host_id)) if (current_zookeeper->tryGet(replica_path, replica_host_id))
@ -290,7 +291,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach,
} }
catch (...) catch (...)
{ {
if (!force_attach) if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
throw; throw;
/// It's server startup, ignore error. /// It's server startup, ignore error.
@ -339,6 +340,8 @@ void DatabaseReplicated::createEmptyLogEntry(const ZooKeeperPtr & current_zookee
bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms) bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms)
{ {
if (!ddl_worker)
return false;
return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms); return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms);
} }
@ -374,21 +377,21 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt
createEmptyLogEntry(current_zookeeper); createEmptyLogEntry(current_zookeeper);
} }
void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool /*force_restore*/, bool force_attach) void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, LoadingStrictnessLevel mode)
{ {
tryConnectToZooKeeperAndInitDatabase(force_attach, /* is_create_query */ !force_attach); tryConnectToZooKeeperAndInitDatabase(mode);
} }
void DatabaseReplicated::loadStoredObjects( void DatabaseReplicated::loadStoredObjects(
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables) ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
{ {
beforeLoadingMetadata(local_context, force_restore, force_attach); beforeLoadingMetadata(local_context, mode);
DatabaseAtomic::loadStoredObjects(local_context, force_restore, force_attach, skip_startup_tables); DatabaseAtomic::loadStoredObjects(local_context, mode, skip_startup_tables);
} }
void DatabaseReplicated::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) void DatabaseReplicated::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{ {
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach); DatabaseAtomic::startupTables(thread_pool, mode);
ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext()); ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
if (is_probably_dropped) if (is_probably_dropped)
return; return;

View File

@ -64,11 +64,11 @@ public:
void drop(ContextPtr /*context*/) override; void drop(ContextPtr /*context*/) override;
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override; void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
void beforeLoadingMetadata(ContextMutablePtr context, bool force_restore, bool force_attach) override; void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override; void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
void shutdown() override; void shutdown() override;
@ -78,7 +78,7 @@ public:
friend struct DatabaseReplicatedTask; friend struct DatabaseReplicatedTask;
friend class DatabaseReplicatedDDLWorker; friend class DatabaseReplicatedDDLWorker;
private: private:
void tryConnectToZooKeeperAndInitDatabase(bool force_attach, bool is_create_query); void tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel mode);
bool createDatabaseNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper); bool createDatabaseNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);
void createReplicaNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper); void createReplicaNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);

View File

@ -35,7 +35,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
chassert(!database->is_probably_dropped); chassert(!database->is_probably_dropped);
auto zookeeper = getAndSetZooKeeper(); auto zookeeper = getAndSetZooKeeper();
if (database->is_readonly) if (database->is_readonly)
database->tryConnectToZooKeeperAndInitDatabase(/* force_attach */ false, /* is_create_query */ false); database->tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel::ATTACH);
initializeReplication(); initializeReplication();
initialized = true; initialized = true;
return true; return true;

View File

@ -1,12 +1,13 @@
#pragma once #pragma once
#include <base/types.h> #include <Core/UUID.h>
#include <Databases/LoadingStrictnessLevel.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/IAST_fwd.h> #include <Parsers/IAST_fwd.h>
#include <Storages/IStorage_fwd.h> #include <Storages/IStorage_fwd.h>
#include <Interpreters/Context_fwd.h> #include <base/types.h>
#include <Common/Exception.h> #include <Common/Exception.h>
#include <Common/ThreadPool.h> #include <Common/ThreadPool.h>
#include <Core/UUID.h>
#include <ctime> #include <ctime>
#include <functional> #include <functional>
@ -132,18 +133,15 @@ public:
/// You can call only once, right after the object is created. /// You can call only once, right after the object is created.
virtual void loadStoredObjects( /// NOLINT virtual void loadStoredObjects( /// NOLINT
ContextMutablePtr /*context*/, ContextMutablePtr /*context*/,
bool /*force_restore*/, LoadingStrictnessLevel /*mode*/,
bool /*force_attach*/ = false, bool /* skip_startup_tables */)
bool /* skip_startup_tables */ = false)
{ {
} }
virtual bool supportsLoadingInTopologicalOrder() const { return false; } virtual bool supportsLoadingInTopologicalOrder() const { return false; }
virtual void beforeLoadingMetadata( virtual void beforeLoadingMetadata(
ContextMutablePtr /*context*/, ContextMutablePtr /*context*/, LoadingStrictnessLevel /*mode*/)
bool /*force_restore*/,
bool /*force_attach*/)
{ {
} }
@ -152,12 +150,13 @@ public:
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented");
} }
virtual void loadTableFromMetadata(ContextMutablePtr /*local_context*/, const String & /*file_path*/, const QualifiedTableName & /*name*/, const ASTPtr & /*ast*/, bool /*force_restore*/) virtual void loadTableFromMetadata(ContextMutablePtr /*local_context*/, const String & /*file_path*/, const QualifiedTableName & /*name*/, const ASTPtr & /*ast*/,
LoadingStrictnessLevel /*mode*/)
{ {
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented"); throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented");
} }
virtual void startupTables(ThreadPool & /*thread_pool*/, bool /*force_restore*/, bool /*force_attach*/) {} virtual void startupTables(ThreadPool & /*thread_pool*/, LoadingStrictnessLevel /*mode*/) {}
/// Check the existence of the table in memory (attached). /// Check the existence of the table in memory (attached).
virtual bool isTableExist(const String & name, ContextPtr context) const = 0; virtual bool isTableExist(const String & name, ContextPtr context) const = 0;

View File

@ -0,0 +1,27 @@
#include <Databases/LoadingStrictnessLevel.h>
namespace DB
{
LoadingStrictnessLevel getLoadingStrictnessLevel(bool attach, bool force_attach, bool force_restore)
{
if (force_restore)
{
assert(attach);
assert(force_attach);
return LoadingStrictnessLevel::FORCE_RESTORE;
}
if (force_attach)
{
assert(attach);
return LoadingStrictnessLevel::FORCE_ATTACH;
}
if (attach)
return LoadingStrictnessLevel::ATTACH;
return LoadingStrictnessLevel::CREATE;
}
}

View File

@ -0,0 +1,16 @@
#pragma once
namespace DB
{
enum class LoadingStrictnessLevel
{
CREATE = 0,
ATTACH = 1,
FORCE_ATTACH = 2,
FORCE_RESTORE = 3,
};
LoadingStrictnessLevel getLoadingStrictnessLevel(bool attach, bool force_attach, bool force_restore);
}

View File

@ -63,11 +63,11 @@ void DatabaseMaterializedMySQL::setException(const std::exception_ptr & exceptio
exception = exception_; exception = exception_;
} }
void DatabaseMaterializedMySQL::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) void DatabaseMaterializedMySQL::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{ {
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach); DatabaseAtomic::startupTables(thread_pool, mode);
if (!force_attach) if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
materialize_thread.assertMySQLAvailable(); materialize_thread.assertMySQLAvailable();
materialize_thread.startSynchronization(); materialize_thread.startSynchronization();

View File

@ -48,7 +48,7 @@ protected:
public: public:
String getEngineName() const override { return "MaterializedMySQL"; } String getEngineName() const override { return "MaterializedMySQL"; }
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override; void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override; void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override;

View File

@ -398,7 +398,7 @@ String DatabaseMySQL::getMetadataPath() const
return metadata_path; return metadata_path;
} }
void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /*force_attach*/, bool /* skip_startup_tables */) void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
{ {
std::lock_guard<std::mutex> lock{mutex}; std::lock_guard<std::mutex> lock{mutex};

View File

@ -76,7 +76,7 @@ public:
void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override; void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override;
void loadStoredObjects(ContextMutablePtr, bool, bool force_attach, bool skip_startup_tables) override; void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
StoragePtr detachTable(ContextPtr context, const String & table_name) override; StoragePtr detachTable(ContextPtr context, const String & table_name) override;

View File

@ -125,9 +125,9 @@ void DatabaseMaterializedPostgreSQL::startSynchronization()
} }
void DatabaseMaterializedPostgreSQL::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) void DatabaseMaterializedPostgreSQL::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
{ {
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach); DatabaseAtomic::startupTables(thread_pool, mode);
startup_task->activateAndSchedule(); startup_task->activateAndSchedule();
} }

View File

@ -40,7 +40,7 @@ public:
String getMetadataPath() const override { return metadata_path; } String getMetadataPath() const override { return metadata_path; }
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override; void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
DatabaseTablesIteratorPtr DatabaseTablesIteratorPtr
getTablesIterator(ContextPtr context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override; getTablesIterator(ContextPtr context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override;

View File

@ -290,7 +290,7 @@ void DatabasePostgreSQL::drop(ContextPtr /*context*/)
} }
void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool, bool /*force_attach*/, bool /* skip_startup_tables */) void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
{ {
{ {
std::lock_guard<std::mutex> lock{mutex}; std::lock_guard<std::mutex> lock{mutex};

View File

@ -45,7 +45,7 @@ public:
bool empty() const override; bool empty() const override;
void loadStoredObjects(ContextMutablePtr, bool, bool force_attach, bool skip_startup_tables) override; void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;

View File

@ -62,11 +62,10 @@ void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, Atomic
} }
} }
TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_, bool force_attach_) TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_)
: global_context(global_context_) : global_context(global_context_)
, databases(std::move(databases_)) , databases(std::move(databases_))
, force_restore(force_restore_) , strictness_mode(strictness_mode_)
, force_attach(force_attach_)
{ {
metadata.default_database = global_context->getCurrentDatabase(); metadata.default_database = global_context->getCurrentDatabase();
log = &Poco::Logger::get("TablesLoader"); log = &Poco::Logger::get("TablesLoader");
@ -83,7 +82,7 @@ void TablesLoader::loadTables()
if (need_resolve_dependencies && database.second->supportsLoadingInTopologicalOrder()) if (need_resolve_dependencies && database.second->supportsLoadingInTopologicalOrder())
databases_to_load.push_back(database.first); databases_to_load.push_back(database.first);
else else
database.second->loadStoredObjects(global_context, force_restore, force_attach, true); database.second->loadStoredObjects(global_context, strictness_mode, /* skip_startup_tables */ true);
} }
if (databases_to_load.empty()) if (databases_to_load.empty())
@ -92,8 +91,9 @@ void TablesLoader::loadTables()
/// Read and parse metadata from Ordinary, Atomic, Materialized*, Replicated, etc databases. Build dependency graph. /// Read and parse metadata from Ordinary, Atomic, Materialized*, Replicated, etc databases. Build dependency graph.
for (auto & database_name : databases_to_load) for (auto & database_name : databases_to_load)
{ {
databases[database_name]->beforeLoadingMetadata(global_context, force_restore, force_attach); databases[database_name]->beforeLoadingMetadata(global_context, strictness_mode);
databases[database_name]->loadTablesMetadata(global_context, metadata, force_attach); bool is_startup = LoadingStrictnessLevel::FORCE_ATTACH <= strictness_mode;
databases[database_name]->loadTablesMetadata(global_context, metadata, is_startup);
} }
LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec", LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec",
@ -119,7 +119,7 @@ void TablesLoader::startupTables()
{ {
/// Startup tables after all tables are loaded. Background tasks (merges, mutations, etc) may slow down data parts loading. /// Startup tables after all tables are loaded. Background tasks (merges, mutations, etc) may slow down data parts loading.
for (auto & database : databases) for (auto & database : databases)
database.second->startupTables(pool, force_restore, force_attach); database.second->startupTables(pool, strictness_mode);
} }
@ -253,7 +253,7 @@ void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level
pool.scheduleOrThrowOnError([this, load_context, total_tables, &table_name]() pool.scheduleOrThrowOnError([this, load_context, total_tables, &table_name]()
{ {
const auto & path_and_query = metadata.parsed_tables[table_name]; const auto & path_and_query = metadata.parsed_tables[table_name];
databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, force_restore); databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, strictness_mode);
logAboutProgress(log, ++tables_processed, total_tables, stopwatch); logAboutProgress(log, ++tables_processed, total_tables, stopwatch);
}); });
} }

View File

@ -1,14 +1,15 @@
#pragma once #pragma once
#include <Core/Types.h>
#include <Core/QualifiedTableName.h>
#include <Parsers/IAST_fwd.h>
#include <Interpreters/Context_fwd.h>
#include <Common/ThreadPool.h>
#include <Common/Stopwatch.h>
#include <map> #include <map>
#include <mutex>
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <mutex> #include <Core/QualifiedTableName.h>
#include <Core/Types.h>
#include <Databases/LoadingStrictnessLevel.h>
#include <Interpreters/Context_fwd.h>
#include <Parsers/IAST_fwd.h>
#include <Common/Stopwatch.h>
#include <Common/ThreadPool.h>
namespace Poco namespace Poco
{ {
@ -78,7 +79,7 @@ class TablesLoader
public: public:
using Databases = std::map<String, DatabasePtr>; using Databases = std::map<String, DatabasePtr>;
TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_ = false, bool force_attach_ = false); TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_);
TablesLoader() = delete; TablesLoader() = delete;
void loadTables(); void loadTables();
@ -87,8 +88,7 @@ public:
private: private:
ContextMutablePtr global_context; ContextMutablePtr global_context;
Databases databases; Databases databases;
bool force_restore; LoadingStrictnessLevel strictness_mode;
bool force_attach;
Strings databases_to_load; Strings databases_to_load;
ParsedTablesMetadata metadata; ParsedTablesMetadata metadata;

View File

@ -249,6 +249,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path); bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path);
bool need_lock_uuid = internal || need_write_metadata; bool need_lock_uuid = internal || need_write_metadata;
auto mode = getLoadingStrictnessLevel(create.attach, force_attach, has_force_restore_data_flag);
/// Lock uuid, so we will known it's already in use. /// Lock uuid, so we will known it's already in use.
/// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach); /// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach);
@ -303,8 +304,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
if (!load_database_without_tables) if (!load_database_without_tables)
{ {
/// We use global context here, because storages lifetime is bigger than query context lifetime /// We use global context here, because storages lifetime is bigger than query context lifetime
TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, has_force_restore_data_flag, create.attach && force_attach}; //-V560 TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode}; //-V560
loader.loadTables(); loader.loadTables();
loader.startupTables(); loader.startupTables();
} }

View File

@ -760,6 +760,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query) void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
{ {
const auto database_name = query.getDatabase(); const auto database_name = query.getDatabase();
auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, "");
auto database = DatabaseCatalog::instance().getDatabase(database_name); auto database = DatabaseCatalog::instance().getDatabase(database_name);
if (auto * ptr = typeid_cast<DatabaseReplicated *>(database.get())) if (auto * ptr = typeid_cast<DatabaseReplicated *>(database.get()))
@ -767,8 +768,7 @@ void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
LOG_TRACE(log, "Synchronizing entries in the database replica's (name: {}) queue with the log", database_name); LOG_TRACE(log, "Synchronizing entries in the database replica's (name: {}) queue with the log", database_name);
if (!ptr->waitForReplicaToProcessAllEntries(getContext()->getSettingsRef().receive_timeout.totalMilliseconds())) if (!ptr->waitForReplicaToProcessAllEntries(getContext()->getSettingsRef().receive_timeout.totalMilliseconds()))
{ {
LOG_ERROR(log, "SYNC DATABASE REPLICA {}: Timed out!", database_name); throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: database is readonly or command timed out. " \
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: command timed out. " \
"See the 'receive_timeout' setting", database_name); "See the 'receive_timeout' setting", database_name);
} }
LOG_TRACE(log, "SYNC DATABASE REPLICA {}: OK", database_name); LOG_TRACE(log, "SYNC DATABASE REPLICA {}: OK", database_name);

View File

@ -38,6 +38,7 @@ static void executeCreateQuery(
ContextMutablePtr context, ContextMutablePtr context,
const String & database, const String & database,
const String & file_name, const String & file_name,
bool create,
bool has_force_restore_data_flag) bool has_force_restore_data_flag)
{ {
ParserCreateQuery parser; ParserCreateQuery parser;
@ -49,8 +50,11 @@ static void executeCreateQuery(
InterpreterCreateQuery interpreter(ast, context); InterpreterCreateQuery interpreter(ast, context);
interpreter.setInternal(true); interpreter.setInternal(true);
interpreter.setForceAttach(true); if (!create)
interpreter.setForceRestoreData(has_force_restore_data_flag); {
interpreter.setForceAttach(true);
interpreter.setForceRestoreData(has_force_restore_data_flag);
}
interpreter.setLoadDatabaseWithoutTables(true); interpreter.setLoadDatabaseWithoutTables(true);
interpreter.execute(); interpreter.execute();
} }
@ -86,7 +90,7 @@ static void loadDatabase(
try try
{ {
executeCreateQuery(database_attach_query, context, database, database_metadata_file, force_restore_data); executeCreateQuery(database_attach_query, context, database, database_metadata_file, /* create */ true, force_restore_data);
} }
catch (Exception & e) catch (Exception & e)
{ {
@ -173,7 +177,8 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
loaded_databases.insert({name, DatabaseCatalog::instance().getDatabase(name)}); loaded_databases.insert({name, DatabaseCatalog::instance().getDatabase(name)});
} }
TablesLoader loader{context, std::move(loaded_databases), has_force_restore_data_flag, /* force_attach */ true}; auto mode = getLoadingStrictnessLevel(/* attach */ true, /* force_attach */ true, has_force_restore_data_flag);
TablesLoader loader{context, std::move(loaded_databases), mode};
loader.loadTables(); loader.loadTables();
loader.startupTables(); loader.startupTables();
@ -207,7 +212,7 @@ static void loadSystemDatabaseImpl(ContextMutablePtr context, const String & dat
database_create_query += database_name; database_create_query += database_name;
database_create_query += " ENGINE="; database_create_query += " ENGINE=";
database_create_query += default_engine; database_create_query += default_engine;
executeCreateQuery(database_create_query, context, database_name, "<no file>", true); executeCreateQuery(database_create_query, context, database_name, "<no file>", true, true);
} }
} }
@ -315,7 +320,7 @@ void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const Datab
{ {
{DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::instance().getSystemDatabase()}, {DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::instance().getSystemDatabase()},
}; };
TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true}; TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
loader.loadTables(); loader.loadTables();
/// Will startup tables usual way /// Will startup tables usual way
@ -331,7 +336,7 @@ void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const Datab
void startupSystemTables() void startupSystemTables()
{ {
ThreadPool pool; ThreadPool pool;
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, /* force_restore */ true, /* force_attach */ true); DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE);
} }
void loadMetadataSystem(ContextMutablePtr context) void loadMetadataSystem(ContextMutablePtr context)
@ -346,7 +351,7 @@ void loadMetadataSystem(ContextMutablePtr context)
{DatabaseCatalog::INFORMATION_SCHEMA, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA)}, {DatabaseCatalog::INFORMATION_SCHEMA, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA)},
{DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)}, {DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)},
}; };
TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true}; TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
loader.loadTables(); loader.loadTables();
/// Will startup tables in system database after all databases are loaded. /// Will startup tables in system database after all databases are loaded.
} }