mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-18 21:51:57 +00:00
fix
This commit is contained in:
parent
e295fd5090
commit
6e77155dc8
@ -416,9 +416,9 @@ UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const
|
||||
return UUIDHelpers::Nil;
|
||||
}
|
||||
|
||||
void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool force_restore, bool /*force_attach*/)
|
||||
void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, LoadingStrictnessLevel mode)
|
||||
{
|
||||
if (!force_restore)
|
||||
if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
|
||||
return;
|
||||
|
||||
/// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken
|
||||
@ -435,17 +435,17 @@ void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool f
|
||||
}
|
||||
|
||||
void DatabaseAtomic::loadStoredObjects(
|
||||
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables)
|
||||
ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
|
||||
{
|
||||
beforeLoadingMetadata(local_context, force_restore, force_attach);
|
||||
DatabaseOrdinary::loadStoredObjects(local_context, force_restore, force_attach, skip_startup_tables);
|
||||
beforeLoadingMetadata(local_context, mode);
|
||||
DatabaseOrdinary::loadStoredObjects(local_context, mode, skip_startup_tables);
|
||||
}
|
||||
|
||||
void DatabaseAtomic::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
|
||||
void DatabaseAtomic::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
|
||||
{
|
||||
DatabaseOrdinary::startupTables(thread_pool, force_restore, force_attach);
|
||||
DatabaseOrdinary::startupTables(thread_pool, mode);
|
||||
|
||||
if (!force_restore)
|
||||
if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
|
||||
return;
|
||||
|
||||
NameToPathMap table_names;
|
||||
|
@ -47,11 +47,11 @@ public:
|
||||
|
||||
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
|
||||
|
||||
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
|
||||
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
|
||||
|
||||
void beforeLoadingMetadata(ContextMutablePtr context, bool force_restore, bool force_attach) override;
|
||||
void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
|
||||
|
||||
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
|
||||
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
|
||||
|
||||
/// Atomic database cannot be detached if there is detached table which still in use
|
||||
void assertCanBeDetached(bool cleanup) override;
|
||||
|
@ -38,7 +38,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_,
|
||||
|
||||
|
||||
void DatabaseLazy::loadStoredObjects(
|
||||
ContextMutablePtr local_context, bool /* force_restore */, bool /*force_attach*/, bool /* skip_startup_tables */)
|
||||
ContextMutablePtr local_context, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
|
||||
{
|
||||
iterateMetadataFiles(local_context, [this, &local_context](const String & file_name)
|
||||
{
|
||||
|
@ -26,7 +26,7 @@ public:
|
||||
|
||||
bool canContainDistributedTables() const override { return false; }
|
||||
|
||||
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
|
||||
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
|
||||
|
||||
void createTable(
|
||||
ContextPtr context,
|
||||
|
@ -81,7 +81,7 @@ DatabaseOrdinary::DatabaseOrdinary(
|
||||
}
|
||||
|
||||
void DatabaseOrdinary::loadStoredObjects(
|
||||
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables)
|
||||
ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
|
||||
{
|
||||
/** Tables load faster if they are loaded in sorted (by name) order.
|
||||
* Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
|
||||
@ -89,6 +89,7 @@ void DatabaseOrdinary::loadStoredObjects(
|
||||
*/
|
||||
|
||||
ParsedTablesMetadata metadata;
|
||||
bool force_attach = LoadingStrictnessLevel::FORCE_ATTACH <= mode;
|
||||
loadTablesMetadata(local_context, metadata, force_attach);
|
||||
|
||||
size_t total_tables = metadata.parsed_tables.size() - metadata.total_dictionaries;
|
||||
@ -118,7 +119,7 @@ void DatabaseOrdinary::loadStoredObjects(
|
||||
{
|
||||
pool.scheduleOrThrowOnError([&]()
|
||||
{
|
||||
loadTableFromMetadata(local_context, path, name, ast, force_restore);
|
||||
loadTableFromMetadata(local_context, path, name, ast, mode);
|
||||
|
||||
/// Messages, so that it's not boring to wait for the server to load for a long time.
|
||||
logAboutProgress(log, ++dictionaries_processed, metadata.total_dictionaries, watch);
|
||||
@ -140,7 +141,7 @@ void DatabaseOrdinary::loadStoredObjects(
|
||||
{
|
||||
pool.scheduleOrThrowOnError([&]()
|
||||
{
|
||||
loadTableFromMetadata(local_context, path, name, ast, force_restore);
|
||||
loadTableFromMetadata(local_context, path, name, ast, mode);
|
||||
|
||||
/// Messages, so that it's not boring to wait for the server to load for a long time.
|
||||
logAboutProgress(log, ++tables_processed, total_tables, watch);
|
||||
@ -153,7 +154,7 @@ void DatabaseOrdinary::loadStoredObjects(
|
||||
if (!skip_startup_tables)
|
||||
{
|
||||
/// After all tables was basically initialized, startup them.
|
||||
startupTables(pool, force_restore, force_attach);
|
||||
startupTables(pool, mode);
|
||||
}
|
||||
}
|
||||
|
||||
@ -238,7 +239,8 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
|
||||
TSA_SUPPRESS_WARNING_FOR_READ(database_name), tables_in_database, dictionaries_in_database);
|
||||
}
|
||||
|
||||
void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore)
|
||||
void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast,
|
||||
LoadingStrictnessLevel mode)
|
||||
{
|
||||
assert(name.database == TSA_SUPPRESS_WARNING_FOR_READ(database_name));
|
||||
const auto & create_query = ast->as<const ASTCreateQuery &>();
|
||||
@ -248,11 +250,10 @@ void DatabaseOrdinary::loadTableFromMetadata(ContextMutablePtr local_context, co
|
||||
create_query,
|
||||
*this,
|
||||
name.database,
|
||||
file_path,
|
||||
force_restore);
|
||||
file_path, LoadingStrictnessLevel::FORCE_RESTORE <= mode);
|
||||
}
|
||||
|
||||
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, bool /*force_restore*/, bool /*force_attach*/)
|
||||
void DatabaseOrdinary::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel /*mode*/)
|
||||
{
|
||||
LOG_INFO(log, "Starting up tables.");
|
||||
|
||||
|
@ -21,15 +21,16 @@ public:
|
||||
|
||||
String getEngineName() const override { return "Ordinary"; }
|
||||
|
||||
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
|
||||
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
|
||||
|
||||
bool supportsLoadingInTopologicalOrder() const override { return true; }
|
||||
|
||||
void loadTablesMetadata(ContextPtr context, ParsedTablesMetadata & metadata, bool is_startup) override;
|
||||
|
||||
void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast, bool force_restore) override;
|
||||
void loadTableFromMetadata(ContextMutablePtr local_context, const String & file_path, const QualifiedTableName & name, const ASTPtr & ast,
|
||||
LoadingStrictnessLevel mode) override;
|
||||
|
||||
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
|
||||
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
|
||||
|
||||
void alterTable(
|
||||
ContextPtr context,
|
||||
|
@ -232,7 +232,7 @@ void DatabaseReplicated::fillClusterAuthInfo(String collection_name, const Poco:
|
||||
cluster_auth_info.cluster_secure_connection = config_ref.getBool(config_prefix + ".cluster_secure_connection", false);
|
||||
}
|
||||
|
||||
void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach, bool is_create_query)
|
||||
void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel mode)
|
||||
{
|
||||
try
|
||||
{
|
||||
@ -250,6 +250,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach,
|
||||
}
|
||||
|
||||
replica_path = fs::path(zookeeper_path) / "replicas" / getFullReplicaName();
|
||||
bool is_create_query = mode == LoadingStrictnessLevel::CREATE;
|
||||
|
||||
String replica_host_id;
|
||||
if (current_zookeeper->tryGet(replica_path, replica_host_id))
|
||||
@ -290,7 +291,7 @@ void DatabaseReplicated::tryConnectToZooKeeperAndInitDatabase(bool force_attach,
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (!force_attach)
|
||||
if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
|
||||
throw;
|
||||
|
||||
/// It's server startup, ignore error.
|
||||
@ -339,6 +340,8 @@ void DatabaseReplicated::createEmptyLogEntry(const ZooKeeperPtr & current_zookee
|
||||
|
||||
bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms)
|
||||
{
|
||||
if (!ddl_worker)
|
||||
return false;
|
||||
return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms);
|
||||
}
|
||||
|
||||
@ -374,21 +377,21 @@ void DatabaseReplicated::createReplicaNodesInZooKeeper(const zkutil::ZooKeeperPt
|
||||
createEmptyLogEntry(current_zookeeper);
|
||||
}
|
||||
|
||||
void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, bool /*force_restore*/, bool force_attach)
|
||||
void DatabaseReplicated::beforeLoadingMetadata(ContextMutablePtr /*context*/, LoadingStrictnessLevel mode)
|
||||
{
|
||||
tryConnectToZooKeeperAndInitDatabase(force_attach, /* is_create_query */ !force_attach);
|
||||
tryConnectToZooKeeperAndInitDatabase(mode);
|
||||
}
|
||||
|
||||
void DatabaseReplicated::loadStoredObjects(
|
||||
ContextMutablePtr local_context, bool force_restore, bool force_attach, bool skip_startup_tables)
|
||||
ContextMutablePtr local_context, LoadingStrictnessLevel mode, bool skip_startup_tables)
|
||||
{
|
||||
beforeLoadingMetadata(local_context, force_restore, force_attach);
|
||||
DatabaseAtomic::loadStoredObjects(local_context, force_restore, force_attach, skip_startup_tables);
|
||||
beforeLoadingMetadata(local_context, mode);
|
||||
DatabaseAtomic::loadStoredObjects(local_context, mode, skip_startup_tables);
|
||||
}
|
||||
|
||||
void DatabaseReplicated::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
|
||||
void DatabaseReplicated::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
|
||||
{
|
||||
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach);
|
||||
DatabaseAtomic::startupTables(thread_pool, mode);
|
||||
ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
|
||||
if (is_probably_dropped)
|
||||
return;
|
||||
|
@ -64,11 +64,11 @@ public:
|
||||
|
||||
void drop(ContextPtr /*context*/) override;
|
||||
|
||||
void loadStoredObjects(ContextMutablePtr context, bool force_restore, bool force_attach, bool skip_startup_tables) override;
|
||||
void loadStoredObjects(ContextMutablePtr context, LoadingStrictnessLevel mode, bool skip_startup_tables) override;
|
||||
|
||||
void beforeLoadingMetadata(ContextMutablePtr context, bool force_restore, bool force_attach) override;
|
||||
void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
|
||||
|
||||
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
|
||||
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
|
||||
|
||||
void shutdown() override;
|
||||
|
||||
@ -78,7 +78,7 @@ public:
|
||||
friend struct DatabaseReplicatedTask;
|
||||
friend class DatabaseReplicatedDDLWorker;
|
||||
private:
|
||||
void tryConnectToZooKeeperAndInitDatabase(bool force_attach, bool is_create_query);
|
||||
void tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel mode);
|
||||
bool createDatabaseNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);
|
||||
void createReplicaNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);
|
||||
|
||||
|
@ -35,7 +35,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
|
||||
chassert(!database->is_probably_dropped);
|
||||
auto zookeeper = getAndSetZooKeeper();
|
||||
if (database->is_readonly)
|
||||
database->tryConnectToZooKeeperAndInitDatabase(/* force_attach */ false, /* is_create_query */ false);
|
||||
database->tryConnectToZooKeeperAndInitDatabase(LoadingStrictnessLevel::ATTACH);
|
||||
initializeReplication();
|
||||
initialized = true;
|
||||
return true;
|
||||
|
@ -1,12 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include <base/types.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <Databases/LoadingStrictnessLevel.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <Storages/IStorage_fwd.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <base/types.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Core/UUID.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <functional>
|
||||
@ -132,18 +133,15 @@ public:
|
||||
/// You can call only once, right after the object is created.
|
||||
virtual void loadStoredObjects( /// NOLINT
|
||||
ContextMutablePtr /*context*/,
|
||||
bool /*force_restore*/,
|
||||
bool /*force_attach*/ = false,
|
||||
bool /* skip_startup_tables */ = false)
|
||||
LoadingStrictnessLevel /*mode*/,
|
||||
bool /* skip_startup_tables */)
|
||||
{
|
||||
}
|
||||
|
||||
virtual bool supportsLoadingInTopologicalOrder() const { return false; }
|
||||
|
||||
virtual void beforeLoadingMetadata(
|
||||
ContextMutablePtr /*context*/,
|
||||
bool /*force_restore*/,
|
||||
bool /*force_attach*/)
|
||||
ContextMutablePtr /*context*/, LoadingStrictnessLevel /*mode*/)
|
||||
{
|
||||
}
|
||||
|
||||
@ -152,12 +150,13 @@ public:
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented");
|
||||
}
|
||||
|
||||
virtual void loadTableFromMetadata(ContextMutablePtr /*local_context*/, const String & /*file_path*/, const QualifiedTableName & /*name*/, const ASTPtr & /*ast*/, bool /*force_restore*/)
|
||||
virtual void loadTableFromMetadata(ContextMutablePtr /*local_context*/, const String & /*file_path*/, const QualifiedTableName & /*name*/, const ASTPtr & /*ast*/,
|
||||
LoadingStrictnessLevel /*mode*/)
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not implemented");
|
||||
}
|
||||
|
||||
virtual void startupTables(ThreadPool & /*thread_pool*/, bool /*force_restore*/, bool /*force_attach*/) {}
|
||||
virtual void startupTables(ThreadPool & /*thread_pool*/, LoadingStrictnessLevel /*mode*/) {}
|
||||
|
||||
/// Check the existence of the table in memory (attached).
|
||||
virtual bool isTableExist(const String & name, ContextPtr context) const = 0;
|
||||
|
27
src/Databases/LoadingStrictnessLevel.cpp
Normal file
27
src/Databases/LoadingStrictnessLevel.cpp
Normal file
@ -0,0 +1,27 @@
|
||||
#include <Databases/LoadingStrictnessLevel.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
LoadingStrictnessLevel getLoadingStrictnessLevel(bool attach, bool force_attach, bool force_restore)
|
||||
{
|
||||
if (force_restore)
|
||||
{
|
||||
assert(attach);
|
||||
assert(force_attach);
|
||||
return LoadingStrictnessLevel::FORCE_RESTORE;
|
||||
}
|
||||
|
||||
if (force_attach)
|
||||
{
|
||||
assert(attach);
|
||||
return LoadingStrictnessLevel::FORCE_ATTACH;
|
||||
}
|
||||
|
||||
if (attach)
|
||||
return LoadingStrictnessLevel::ATTACH;
|
||||
|
||||
return LoadingStrictnessLevel::CREATE;
|
||||
}
|
||||
|
||||
}
|
16
src/Databases/LoadingStrictnessLevel.h
Normal file
16
src/Databases/LoadingStrictnessLevel.h
Normal file
@ -0,0 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
enum class LoadingStrictnessLevel
|
||||
{
|
||||
CREATE = 0,
|
||||
ATTACH = 1,
|
||||
FORCE_ATTACH = 2,
|
||||
FORCE_RESTORE = 3,
|
||||
};
|
||||
|
||||
LoadingStrictnessLevel getLoadingStrictnessLevel(bool attach, bool force_attach, bool force_restore);
|
||||
|
||||
}
|
@ -63,11 +63,11 @@ void DatabaseMaterializedMySQL::setException(const std::exception_ptr & exceptio
|
||||
exception = exception_;
|
||||
}
|
||||
|
||||
void DatabaseMaterializedMySQL::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
|
||||
void DatabaseMaterializedMySQL::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
|
||||
{
|
||||
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach);
|
||||
DatabaseAtomic::startupTables(thread_pool, mode);
|
||||
|
||||
if (!force_attach)
|
||||
if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
|
||||
materialize_thread.assertMySQLAvailable();
|
||||
|
||||
materialize_thread.startSynchronization();
|
||||
|
@ -48,7 +48,7 @@ protected:
|
||||
public:
|
||||
String getEngineName() const override { return "MaterializedMySQL"; }
|
||||
|
||||
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
|
||||
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
|
||||
|
||||
void createTable(ContextPtr context_, const String & name, const StoragePtr & table, const ASTPtr & query) override;
|
||||
|
||||
|
@ -398,7 +398,7 @@ String DatabaseMySQL::getMetadataPath() const
|
||||
return metadata_path;
|
||||
}
|
||||
|
||||
void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, bool, bool /*force_attach*/, bool /* skip_startup_tables */)
|
||||
void DatabaseMySQL::loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
|
||||
{
|
||||
|
||||
std::lock_guard<std::mutex> lock{mutex};
|
||||
|
@ -76,7 +76,7 @@ public:
|
||||
|
||||
void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override;
|
||||
|
||||
void loadStoredObjects(ContextMutablePtr, bool, bool force_attach, bool skip_startup_tables) override;
|
||||
void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
|
||||
|
||||
StoragePtr detachTable(ContextPtr context, const String & table_name) override;
|
||||
|
||||
|
@ -125,9 +125,9 @@ void DatabaseMaterializedPostgreSQL::startSynchronization()
|
||||
}
|
||||
|
||||
|
||||
void DatabaseMaterializedPostgreSQL::startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach)
|
||||
void DatabaseMaterializedPostgreSQL::startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode)
|
||||
{
|
||||
DatabaseAtomic::startupTables(thread_pool, force_restore, force_attach);
|
||||
DatabaseAtomic::startupTables(thread_pool, mode);
|
||||
startup_task->activateAndSchedule();
|
||||
}
|
||||
|
||||
|
@ -40,7 +40,7 @@ public:
|
||||
|
||||
String getMetadataPath() const override { return metadata_path; }
|
||||
|
||||
void startupTables(ThreadPool & thread_pool, bool force_restore, bool force_attach) override;
|
||||
void startupTables(ThreadPool & thread_pool, LoadingStrictnessLevel mode) override;
|
||||
|
||||
DatabaseTablesIteratorPtr
|
||||
getTablesIterator(ContextPtr context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name) const override;
|
||||
|
@ -290,7 +290,7 @@ void DatabasePostgreSQL::drop(ContextPtr /*context*/)
|
||||
}
|
||||
|
||||
|
||||
void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, bool, bool /*force_attach*/, bool /* skip_startup_tables */)
|
||||
void DatabasePostgreSQL::loadStoredObjects(ContextMutablePtr /* context */, LoadingStrictnessLevel /*mode*/, bool /* skip_startup_tables */)
|
||||
{
|
||||
{
|
||||
std::lock_guard<std::mutex> lock{mutex};
|
||||
|
@ -45,7 +45,7 @@ public:
|
||||
|
||||
bool empty() const override;
|
||||
|
||||
void loadStoredObjects(ContextMutablePtr, bool, bool force_attach, bool skip_startup_tables) override;
|
||||
void loadStoredObjects(ContextMutablePtr, LoadingStrictnessLevel /*mode*/, bool skip_startup_tables) override;
|
||||
|
||||
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
|
||||
|
||||
|
@ -62,11 +62,10 @@ void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, Atomic
|
||||
}
|
||||
}
|
||||
|
||||
TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_, bool force_attach_)
|
||||
TablesLoader::TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_)
|
||||
: global_context(global_context_)
|
||||
, databases(std::move(databases_))
|
||||
, force_restore(force_restore_)
|
||||
, force_attach(force_attach_)
|
||||
, strictness_mode(strictness_mode_)
|
||||
{
|
||||
metadata.default_database = global_context->getCurrentDatabase();
|
||||
log = &Poco::Logger::get("TablesLoader");
|
||||
@ -83,7 +82,7 @@ void TablesLoader::loadTables()
|
||||
if (need_resolve_dependencies && database.second->supportsLoadingInTopologicalOrder())
|
||||
databases_to_load.push_back(database.first);
|
||||
else
|
||||
database.second->loadStoredObjects(global_context, force_restore, force_attach, true);
|
||||
database.second->loadStoredObjects(global_context, strictness_mode, /* skip_startup_tables */ true);
|
||||
}
|
||||
|
||||
if (databases_to_load.empty())
|
||||
@ -92,8 +91,9 @@ void TablesLoader::loadTables()
|
||||
/// Read and parse metadata from Ordinary, Atomic, Materialized*, Replicated, etc databases. Build dependency graph.
|
||||
for (auto & database_name : databases_to_load)
|
||||
{
|
||||
databases[database_name]->beforeLoadingMetadata(global_context, force_restore, force_attach);
|
||||
databases[database_name]->loadTablesMetadata(global_context, metadata, force_attach);
|
||||
databases[database_name]->beforeLoadingMetadata(global_context, strictness_mode);
|
||||
bool is_startup = LoadingStrictnessLevel::FORCE_ATTACH <= strictness_mode;
|
||||
databases[database_name]->loadTablesMetadata(global_context, metadata, is_startup);
|
||||
}
|
||||
|
||||
LOG_INFO(log, "Parsed metadata of {} tables in {} databases in {} sec",
|
||||
@ -119,7 +119,7 @@ void TablesLoader::startupTables()
|
||||
{
|
||||
/// Startup tables after all tables are loaded. Background tasks (merges, mutations, etc) may slow down data parts loading.
|
||||
for (auto & database : databases)
|
||||
database.second->startupTables(pool, force_restore, force_attach);
|
||||
database.second->startupTables(pool, strictness_mode);
|
||||
}
|
||||
|
||||
|
||||
@ -253,7 +253,7 @@ void TablesLoader::startLoadingIndependentTables(ThreadPool & pool, size_t level
|
||||
pool.scheduleOrThrowOnError([this, load_context, total_tables, &table_name]()
|
||||
{
|
||||
const auto & path_and_query = metadata.parsed_tables[table_name];
|
||||
databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, force_restore);
|
||||
databases[table_name.database]->loadTableFromMetadata(load_context, path_and_query.path, table_name, path_and_query.ast, strictness_mode);
|
||||
logAboutProgress(log, ++tables_processed, total_tables, stopwatch);
|
||||
});
|
||||
}
|
||||
|
@ -1,14 +1,15 @@
|
||||
#pragma once
|
||||
#include <Core/Types.h>
|
||||
#include <Core/QualifiedTableName.h>
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <mutex>
|
||||
#include <Core/QualifiedTableName.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Databases/LoadingStrictnessLevel.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
|
||||
namespace Poco
|
||||
{
|
||||
@ -78,7 +79,7 @@ class TablesLoader
|
||||
public:
|
||||
using Databases = std::map<String, DatabasePtr>;
|
||||
|
||||
TablesLoader(ContextMutablePtr global_context_, Databases databases_, bool force_restore_ = false, bool force_attach_ = false);
|
||||
TablesLoader(ContextMutablePtr global_context_, Databases databases_, LoadingStrictnessLevel strictness_mode_);
|
||||
TablesLoader() = delete;
|
||||
|
||||
void loadTables();
|
||||
@ -87,8 +88,7 @@ public:
|
||||
private:
|
||||
ContextMutablePtr global_context;
|
||||
Databases databases;
|
||||
bool force_restore;
|
||||
bool force_attach;
|
||||
LoadingStrictnessLevel strictness_mode;
|
||||
|
||||
Strings databases_to_load;
|
||||
ParsedTablesMetadata metadata;
|
||||
|
@ -249,6 +249,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
|
||||
|
||||
bool need_write_metadata = !create.attach || !fs::exists(metadata_file_path);
|
||||
bool need_lock_uuid = internal || need_write_metadata;
|
||||
auto mode = getLoadingStrictnessLevel(create.attach, force_attach, has_force_restore_data_flag);
|
||||
|
||||
/// Lock uuid, so we will known it's already in use.
|
||||
/// We do it when attaching databases on server startup (internal) and on CREATE query (!create.attach);
|
||||
@ -303,8 +304,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
|
||||
|
||||
if (!load_database_without_tables)
|
||||
{
|
||||
|
||||
/// We use global context here, because storages lifetime is bigger than query context lifetime
|
||||
TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, has_force_restore_data_flag, create.attach && force_attach}; //-V560
|
||||
TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode}; //-V560
|
||||
loader.loadTables();
|
||||
loader.startupTables();
|
||||
}
|
||||
|
@ -760,6 +760,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
|
||||
void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
|
||||
{
|
||||
const auto database_name = query.getDatabase();
|
||||
auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, "");
|
||||
auto database = DatabaseCatalog::instance().getDatabase(database_name);
|
||||
|
||||
if (auto * ptr = typeid_cast<DatabaseReplicated *>(database.get()))
|
||||
@ -767,8 +768,7 @@ void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query)
|
||||
LOG_TRACE(log, "Synchronizing entries in the database replica's (name: {}) queue with the log", database_name);
|
||||
if (!ptr->waitForReplicaToProcessAllEntries(getContext()->getSettingsRef().receive_timeout.totalMilliseconds()))
|
||||
{
|
||||
LOG_ERROR(log, "SYNC DATABASE REPLICA {}: Timed out!", database_name);
|
||||
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: command timed out. " \
|
||||
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: database is readonly or command timed out. " \
|
||||
"See the 'receive_timeout' setting", database_name);
|
||||
}
|
||||
LOG_TRACE(log, "SYNC DATABASE REPLICA {}: OK", database_name);
|
||||
|
@ -38,6 +38,7 @@ static void executeCreateQuery(
|
||||
ContextMutablePtr context,
|
||||
const String & database,
|
||||
const String & file_name,
|
||||
bool create,
|
||||
bool has_force_restore_data_flag)
|
||||
{
|
||||
ParserCreateQuery parser;
|
||||
@ -49,8 +50,11 @@ static void executeCreateQuery(
|
||||
|
||||
InterpreterCreateQuery interpreter(ast, context);
|
||||
interpreter.setInternal(true);
|
||||
interpreter.setForceAttach(true);
|
||||
interpreter.setForceRestoreData(has_force_restore_data_flag);
|
||||
if (!create)
|
||||
{
|
||||
interpreter.setForceAttach(true);
|
||||
interpreter.setForceRestoreData(has_force_restore_data_flag);
|
||||
}
|
||||
interpreter.setLoadDatabaseWithoutTables(true);
|
||||
interpreter.execute();
|
||||
}
|
||||
@ -86,7 +90,7 @@ static void loadDatabase(
|
||||
|
||||
try
|
||||
{
|
||||
executeCreateQuery(database_attach_query, context, database, database_metadata_file, force_restore_data);
|
||||
executeCreateQuery(database_attach_query, context, database, database_metadata_file, /* create */ true, force_restore_data);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
@ -173,7 +177,8 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
|
||||
loaded_databases.insert({name, DatabaseCatalog::instance().getDatabase(name)});
|
||||
}
|
||||
|
||||
TablesLoader loader{context, std::move(loaded_databases), has_force_restore_data_flag, /* force_attach */ true};
|
||||
auto mode = getLoadingStrictnessLevel(/* attach */ true, /* force_attach */ true, has_force_restore_data_flag);
|
||||
TablesLoader loader{context, std::move(loaded_databases), mode};
|
||||
loader.loadTables();
|
||||
loader.startupTables();
|
||||
|
||||
@ -207,7 +212,7 @@ static void loadSystemDatabaseImpl(ContextMutablePtr context, const String & dat
|
||||
database_create_query += database_name;
|
||||
database_create_query += " ENGINE=";
|
||||
database_create_query += default_engine;
|
||||
executeCreateQuery(database_create_query, context, database_name, "<no file>", true);
|
||||
executeCreateQuery(database_create_query, context, database_name, "<no file>", true, true);
|
||||
}
|
||||
}
|
||||
|
||||
@ -315,7 +320,7 @@ void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const Datab
|
||||
{
|
||||
{DatabaseCatalog::SYSTEM_DATABASE, DatabaseCatalog::instance().getSystemDatabase()},
|
||||
};
|
||||
TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true};
|
||||
TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
|
||||
loader.loadTables();
|
||||
|
||||
/// Will startup tables usual way
|
||||
@ -331,7 +336,7 @@ void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, const Datab
|
||||
void startupSystemTables()
|
||||
{
|
||||
ThreadPool pool;
|
||||
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, /* force_restore */ true, /* force_attach */ true);
|
||||
DatabaseCatalog::instance().getSystemDatabase()->startupTables(pool, LoadingStrictnessLevel::FORCE_RESTORE);
|
||||
}
|
||||
|
||||
void loadMetadataSystem(ContextMutablePtr context)
|
||||
@ -346,7 +351,7 @@ void loadMetadataSystem(ContextMutablePtr context)
|
||||
{DatabaseCatalog::INFORMATION_SCHEMA, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA)},
|
||||
{DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE, DatabaseCatalog::instance().getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)},
|
||||
};
|
||||
TablesLoader loader{context, databases, /* force_restore */ true, /* force_attach */ true};
|
||||
TablesLoader loader{context, databases, LoadingStrictnessLevel::FORCE_RESTORE};
|
||||
loader.loadTables();
|
||||
/// Will startup tables in system database after all databases are loaded.
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user