diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md index 1dac2d25ea5..c863282efc1 100644 --- a/docs/en/operations/utilities/clickhouse-local.md +++ b/docs/en/operations/utilities/clickhouse-local.md @@ -216,6 +216,7 @@ Arguments: - `--logger.level` — Log level. - `--ignore-error` — do not stop processing if a query failed. - `-c`, `--config-file` — path to configuration file in same format as for ClickHouse server, by default the configuration empty. +- `--no-system-tables` — do not attach system tables. - `--help` — arguments references for `clickhouse-local`. - `-V`, `--version` — print version information and exit. diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index 92712a6f6b2..6f0394a183d 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -45,6 +45,7 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom - `--logger.level` — уровень логирования. - `--ignore-error` — не прекращать обработку если запрос выдал ошибку. - `-c`, `--config-file` — путь к файлу конфигурации. По умолчанию `clickhouse-local` запускается с пустой конфигурацией. Конфигурационный файл имеет тот же формат, что и для сервера ClickHouse, и в нём можно использовать все конфигурационные параметры сервера. Обычно подключение конфигурации не требуется; если требуется установить отдельный параметр, то это можно сделать ключом с именем параметра. +- `--no-system-tables` — запуск без использования системных таблиц. - `--help` — вывод справочной информации о `clickhouse-local`. - `-V`, `--version` — вывод текущей версии и выход. diff --git a/docs/zh/operations/utilities/clickhouse-local.md b/docs/zh/operations/utilities/clickhouse-local.md index e8c9503626b..7428ae06a6e 100644 --- a/docs/zh/operations/utilities/clickhouse-local.md +++ b/docs/zh/operations/utilities/clickhouse-local.md @@ -45,6 +45,7 @@ clickhouse-local --structure "table_structure" --input-format "format_of_incomin - `--logger.level` — 日志级别。 - `--ignore-error` — 当查询失败时,不停止处理。 - `-c`, `--config-file` — 与ClickHouse服务器格式相同配置文件的路径,默认情况下配置为空。 +- `--no-system-tables` — 不附加系统表。 - `--help` — `clickhouse-local`使用帮助信息。 - `-V`, `--version` — 打印版本信息并退出。 diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 8e526812957..fbb64ea1135 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -744,7 +744,7 @@ void LocalServer::processConfig() LOG_DEBUG(log, "Loading metadata from {}", path); auto startup_system_tasks = loadMetadataSystem(global_context); - attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); + attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks); @@ -761,9 +761,9 @@ void LocalServer::processConfig() LOG_DEBUG(log, "Loaded metadata."); } - else + else if (!config().has("no-system-tables")) { - attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); + attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); } @@ -842,6 +842,7 @@ void LocalServer::addOptions(OptionsDescription & options_description) ("logger.log", po::value(), "Log file name") ("logger.level", po::value(), "Log level") + ("no-system-tables", "do not attach system tables (better startup time)") ("path", po::value(), "Storage path") ("only-system-tables", "attach only system tables from specified path") ("top_level_domains_path", po::value(), "Path to lists with custom TLDs") @@ -870,6 +871,8 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp config().setString("table-file", options["file"].as()); if (options.count("structure")) config().setString("table-structure", options["structure"].as()); + if (options.count("no-system-tables")) + config().setBool("no-system-tables", true); if (options.count("only-system-tables")) config().setBool("only-system-tables", true); if (options.count("database")) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 1d78e4aff67..1daa6351c23 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -89,14 +89,15 @@ void DatabaseAtomic::drop(ContextPtr) fs::remove_all(getMetadataPath()); } -void DatabaseAtomic::attachTableUnlocked(ContextPtr local_context, const String & name, const StoragePtr & table, const String & relative_table_path) +void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name, const StoragePtr & table, const String & relative_table_path) { assert(relative_table_path != data_path && !relative_table_path.empty()); DetachedTables not_in_use; + std::lock_guard lock(mutex); not_in_use = cleanupDetachedTables(); auto table_id = table->getStorageID(); assertDetachedTableNotInUse(table_id.uuid); - DatabaseOrdinary::attachTableUnlocked(local_context, name, table, relative_table_path); + DatabaseOrdinary::attachTableUnlocked(name, table); table_name_to_path.emplace(std::make_pair(name, relative_table_path)); } @@ -324,7 +325,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora /// It throws if `table_metadata_path` already exists (it's possible if table was detached) renameNoReplace(table_metadata_tmp_path, table_metadata_path); /// Commit point (a sort of) - DatabaseWithOwnTablesBase::attachTableUnlocked(query_context, query.getTable(), table, /*relative_table_path=*/ {}); /// Should never throw + attachTableUnlocked(query.getTable(), table); /// Should never throw table_name_to_path.emplace(query.getTable(), table_data_path); } catch (...) diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h index 1eb6dd1cbbd..83cb51be1ff 100644 --- a/src/Databases/DatabaseAtomic.h +++ b/src/Databases/DatabaseAtomic.h @@ -38,6 +38,7 @@ public: void dropTable(ContextPtr context, const String & table_name, bool sync) override; void dropTableImpl(ContextPtr context, const String & table_name, bool sync); + void attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) override; StoragePtr detachTable(ContextPtr context, const String & name) override; String getTableDataPath(const String & table_name) const override; @@ -65,8 +66,6 @@ public: void setDetachedTableNotInUseForce(const UUID & uuid) override; protected: - void attachTableUnlocked(ContextPtr local_context, const String & name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override; - void commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path, const String & statement, ContextPtr query_context) override; void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table, const String & table_metadata_tmp_path, const String & table_metadata_path, ContextPtr query_context) override; diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 0a1f6d87199..c6249c68933 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -168,9 +168,10 @@ bool DatabaseLazy::empty() const return tables_cache.empty(); } -void DatabaseLazy::attachTableUnlocked(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &) +void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &) { LOG_DEBUG(log, "Attach table {}.", backQuote(table_name)); + std::lock_guard lock(mutex); time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); auto [it, inserted] = tables_cache.emplace(std::piecewise_construct, diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 370c49557de..2b1b119754d 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -64,15 +64,14 @@ public: DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; + void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override; + StoragePtr detachTable(ContextPtr context, const String & table_name) override; void shutdown() override; ~DatabaseLazy() override; -protected: - void attachTableUnlocked(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override; - private: struct CacheExpirationQueueElement { diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 7be9d5e2cb7..2a7a2ad8ccc 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -33,13 +33,13 @@ DatabaseMemory::DatabaseMemory(const String & name_, ContextPtr context_) } void DatabaseMemory::createTable( - ContextPtr local_context, + ContextPtr /*context*/, const String & table_name, const StoragePtr & table, const ASTPtr & query) { std::lock_guard lock{mutex}; - attachTableUnlocked(local_context, table_name, table, /*relative_table_path=*/ {}); + attachTableUnlocked(table_name, table); /// Clean the query from temporary flags. ASTPtr query_to_store = query; @@ -56,7 +56,7 @@ void DatabaseMemory::createTable( } void DatabaseMemory::dropTable( - ContextPtr local_context, + ContextPtr /*context*/, const String & table_name, bool /*sync*/) { @@ -83,7 +83,7 @@ void DatabaseMemory::dropTable( catch (...) { std::lock_guard lock{mutex}; - attachTableUnlocked(local_context, table_name, table, /*relative_table_path=*/ {}); + attachTableUnlocked(table_name, table); throw; } diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 9e743357805..afc09fbe62a 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -200,7 +199,7 @@ DatabaseWithOwnTablesBase::DatabaseWithOwnTablesBase(const String & name_, const bool DatabaseWithOwnTablesBase::isTableExist(const String & table_name, ContextPtr) const { std::lock_guard lock(mutex); - return tables.find(table_name) != tables.end() || lazy_tables.find(table_name) != lazy_tables.end(); + return tables.find(table_name) != tables.end(); } StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name, ContextPtr) const @@ -212,9 +211,6 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name, Con DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name) const { std::lock_guard lock(mutex); - - loadLazyTables(); - if (!filter_by_table_name) return std::make_unique(tables, database_name); @@ -261,7 +257,13 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n return res; } -void DatabaseWithOwnTablesBase::attachTableUnlocked(ContextPtr, const String & name, const StoragePtr & table, const String &) +void DatabaseWithOwnTablesBase::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &) +{ + std::lock_guard lock(mutex); + attachTableUnlocked(table_name, table); +} + +void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, const StoragePtr & table) { auto table_id = table->getStorageID(); if (table_id.database_name != database_name) @@ -274,7 +276,7 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(ContextPtr, const String & n DatabaseCatalog::instance().addUUIDMapping(table_id.uuid, shared_from_this(), table); } - if (!tables.emplace(name, table).second) + if (!tables.emplace(table_name, table).second) { if (table_id.hasUUID()) DatabaseCatalog::instance().removeUUIDMapping(table_id.uuid); @@ -287,12 +289,6 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(ContextPtr, const String & n CurrentMetrics::add(CurrentMetrics::AttachedTable, 1); } -void DatabaseWithOwnTablesBase::registerLazyTableUnlocked(const String & table_name, LazyTableCreator table_creator, const String & relative_table_path) -{ - if (!lazy_tables.emplace(table_name, std::make_pair(relative_table_path, std::move(table_creator))).second) - throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already registered.", table_name); -} - void DatabaseWithOwnTablesBase::shutdown() { /// You can not hold a lock during shutdown. @@ -393,45 +389,10 @@ void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & cre StoragePtr DatabaseWithOwnTablesBase::tryGetTableNoWait(const String & table_name) const { std::lock_guard lock(mutex); - auto it = tables.find(table_name); if (it != tables.end()) return it->second; - - const auto lazy_it = lazy_tables.find(table_name); - if (lazy_it != lazy_tables.end()) - { - LOG_DEBUG(log, "Attaching lazy table {}", backQuoteIfNeed(table_name)); - auto relative_table_path = lazy_it->second.first; - auto storage = lazy_it->second.second(); - lazy_tables.erase(lazy_it); - (const_cast(this))->attachTableUnlocked(Context::getGlobalContextInstance(), table_name, storage, relative_table_path); - - it = tables.find(table_name); - if (it != tables.end()) - return it->second; - } - return {}; } -void DatabaseWithOwnTablesBase::loadLazyTables() const -{ - if (lazy_tables.empty()) - return; - - ContextPtr global_context = Context::getGlobalContextInstance(); - while (!lazy_tables.empty()) - { - auto lazy_it = lazy_tables.begin(); - - const auto table_name = lazy_it->first; - LOG_DEBUG(log, "Attaching lazy table {}", backQuoteIfNeed(table_name)); - auto relative_table_path = lazy_it->second.first; - auto storage = lazy_it->second.second(); - lazy_tables.erase(lazy_it); - (const_cast(this))->attachTableUnlocked(global_context, table_name, storage, relative_table_path); - } -} - } diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h index 71704b95110..fc67596d3de 100644 --- a/src/Databases/DatabasesCommon.h +++ b/src/Databases/DatabasesCommon.h @@ -30,6 +30,8 @@ public: bool empty() const override; + void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override; + StoragePtr detachTable(ContextPtr context, const String & table_name) override; DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override; @@ -43,19 +45,14 @@ public: protected: Tables tables TSA_GUARDED_BY(mutex); - /// Tables that are attached lazily - mutable LazyTables lazy_tables TSA_GUARDED_BY(mutex); Poco::Logger * log; DatabaseWithOwnTablesBase(const String & name_, const String & logger, ContextPtr context); - void attachTableUnlocked(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override; - void registerLazyTableUnlocked(const String & table_name, LazyTableCreator table_creator, const String & relative_table_path) TSA_REQUIRES(mutex) override; + void attachTableUnlocked(const String & table_name, const StoragePtr & table) TSA_REQUIRES(mutex); StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex); StoragePtr getTableUnlocked(const String & table_name) const TSA_REQUIRES(mutex); StoragePtr tryGetTableNoWait(const String & table_name) const; - - void loadLazyTables() const TSA_REQUIRES(mutex); }; } diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index 8b504b967d2..95fcf0c7939 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -62,20 +62,4 @@ void IDatabase::createTableRestoredFromBackup(const ASTPtr & create_table_query, backQuoteIfNeed(create_table_query->as().getTable())); } -/// Add a table to the database, but do not add it to the metadata. The database may not support this method. -/// -/// Note: ATTACH TABLE statement actually uses createTable method. -void IDatabase::attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) /// NOLINT -{ - std::lock_guard lock(mutex); - attachTableUnlocked(context, name, table, relative_table_path); -} - -void IDatabase::registerLazyTable(ContextPtr, const String & table_name, LazyTableCreator table_creator, const String & relative_table_path) /// NOLINT -{ - std::lock_guard lock(mutex); - registerLazyTableUnlocked(table_name, std::move(table_creator), relative_table_path); -} - - } diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 1bbdc74b31b..15e453371b7 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -125,6 +125,7 @@ public: using DatabaseTablesIteratorPtr = std::unique_ptr; + /** Database engine. * It is responsible for: * - initialization of set of known tables and dictionaries; @@ -137,10 +138,6 @@ using DatabaseTablesIteratorPtr = std::unique_ptr; class IDatabase : public std::enable_shared_from_this { public: - using LazyTableCreator = std::function; - /// Map{table_name, Pair{relative_table_path, LazyTableCreator}} - using LazyTables = std::map>; - IDatabase() = delete; explicit IDatabase(String database_name_); @@ -272,17 +269,11 @@ public: /// Add a table to the database, but do not add it to the metadata. The database may not support this method. /// - /// @param relative_table_path - only for Atomic engine - /// - /// Note: - /// - ATTACH TABLE statement actually uses createTable method. - /// - Instead of overriding this method you should override attachTableUnlocked() - /// (This method is only for DatabasesOverlay to override) - virtual void attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path = {}); /// NOLINT - - /// Register tables lazily (attach will be done only when the table will be used) instead of attaching it. - /// This is needed to improve startup time of clickhouse-local. - virtual void registerLazyTable(ContextPtr context, const String & table_name, LazyTableCreator table_creator, const String & relative_table_path = {}); + /// Note: ATTACH TABLE statement actually uses createTable method. + virtual void attachTable(ContextPtr /* context */, const String & /*name*/, const StoragePtr & /*table*/, [[maybe_unused]] const String & relative_table_path = {}) /// NOLINT + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no ATTACH TABLE query for Database{}", getEngineName()); + } /// Forget about the table without deleting it, and return it. The database may not support this method. virtual StoragePtr detachTable(ContextPtr /* context */, const String & /*name*/) @@ -439,16 +430,6 @@ protected: return nullptr; } - virtual void attachTableUnlocked(ContextPtr /*context*/, const String & /*name*/, const StoragePtr & /*table*/, const String & /*relative_table_path*/ = {}) TSA_REQUIRES(mutex) /// NOLINT - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no ATTACH TABLE query for Database{}", getEngineName()); - } - - virtual void registerLazyTableUnlocked(const String & /* table_name */, LazyTableCreator /* table_creator */, const String & /* relative_table_path */) TSA_REQUIRES(mutex) /// NOLINT - { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There lazy table initialization support for Database{}", getEngineName()); - } - mutable std::mutex mutex; String database_name TSA_GUARDED_BY(mutex); String comment TSA_GUARDED_BY(mutex); diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 965a3ac9965..a31e74cc7ae 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -101,10 +101,10 @@ void DatabaseMaterializedMySQL::dropTable(ContextPtr context_, const String & na DatabaseAtomic::dropTable(context_, name, sync); } -void DatabaseMaterializedMySQL::attachTableUnlocked(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) +void DatabaseMaterializedMySQL::attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) { checkIsInternalQuery(context_, "ATTACH TABLE"); - DatabaseAtomic::attachTableUnlocked(context_, name, table, relative_table_path); + DatabaseAtomic::attachTable(context_, name, table, relative_table_path); } StoragePtr DatabaseMaterializedMySQL::detachTable(ContextPtr context_, const String & name) diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.h b/src/Databases/MySQL/DatabaseMaterializedMySQL.h index 0744b79257f..895498723fd 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.h +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.h @@ -48,8 +48,6 @@ protected: LoadTaskPtr startup_mysql_database_task; - void attachTableUnlocked(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override; - public: String getEngineName() const override { return "MaterializedMySQL"; } @@ -60,6 +58,8 @@ public: void dropTable(ContextPtr context_, const String & name, bool sync) override; + void attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) override; + StoragePtr detachTable(ContextPtr context_, const String & name) override; void renameTable(ContextPtr context_, const String & name, IDatabase & to_database, const String & to_name, bool exchange, bool dictionary) override; diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 42bceaa0e08..7d2ed7a9662 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -361,8 +361,10 @@ void DatabaseMySQL::cleanOutdatedTables() } } -void DatabaseMySQL::attachTableUnlocked(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &) +void DatabaseMySQL::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &) { + std::lock_guard lock{mutex}; + if (!local_tables_cache.contains(table_name)) throw Exception(ErrorCodes::UNKNOWN_TABLE, "Cannot attach table {}.{} because it does not exist.", backQuoteIfNeed(database_name), backQuoteIfNeed(table_name)); diff --git a/src/Databases/MySQL/DatabaseMySQL.h b/src/Databases/MySQL/DatabaseMySQL.h index 33bf26059c8..e5b1f434d2f 100644 --- a/src/Databases/MySQL/DatabaseMySQL.h +++ b/src/Databases/MySQL/DatabaseMySQL.h @@ -84,9 +84,9 @@ public: void dropTable(ContextPtr context, const String & table_name, bool sync) override; -protected: - void attachTableUnlocked(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) TSA_REQUIRES(mutex) override; + void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override; +protected: ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override; private: diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 4f0f85bc787..24f04c16029 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -216,8 +216,10 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, ContextPtr } -void DatabasePostgreSQL::attachTableUnlocked(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &) +void DatabasePostgreSQL::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &) { + std::lock_guard lock{mutex}; + if (!checkPostgresTable(table_name)) throw Exception(ErrorCodes::UNKNOWN_TABLE, "Cannot attach PostgreSQL table {} because it does not exist in PostgreSQL (database: {})", diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index 30d0070b2d7..d731e06649b 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -54,14 +54,13 @@ public: void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override; void dropTable(ContextPtr, const String & table_name, bool sync) override; + void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override; StoragePtr detachTable(ContextPtr context, const String & table_name) override; void drop(ContextPtr /*context*/) override; void shutdown() override; protected: - void attachTableUnlocked(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) TSA_REQUIRES(mutex) override; - ASTPtr getCreateTableQueryImpl(const String & table_name, ContextPtr context, bool throw_on_error) const override; private: diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 2d7e728bbba..ca49a546b94 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -108,75 +108,72 @@ namespace DB { -template void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database) { - attachLazyOrNot(context, system_database, "one"); - attachLazyOrNot(context, system_database, "numbers", false); - attachLazyOrNot(context, system_database, "numbers_mt", true); - attachLazyOrNot(context, system_database, "zeros", false); - attachLazyOrNot(context, system_database, "zeros_mt", true); - attachLazyOrNot(context, system_database, "databases"); - attachLazyOrNot(context, system_database, "tables"); - attachLazyOrNot(context, system_database, "columns"); - attachLazyOrNot(context, system_database, "functions"); - attachLazyOrNot(context, system_database, "events"); - attachLazyOrNot(context, system_database, "settings"); - attachLazyOrNot(context, system_database, "server_settings"); - attachLazyOrNot(context, system_database, "settings_changes"); - attachLazyOrNot>(context, system_database, "merge_tree_settings"); - attachLazyOrNot>(context, system_database, "replicated_merge_tree_settings"); - attachLazyOrNot(context, system_database, "build_options"); - attachLazyOrNot(context, system_database, "formats"); - attachLazyOrNot(context, system_database, "table_functions"); - attachLazyOrNot(context, system_database, "aggregate_function_combinators"); - attachLazyOrNot(context, system_database, "data_type_families"); - attachLazyOrNot(context, system_database, "collations"); - attachLazyOrNot(context, system_database, "table_engines"); - attachLazyOrNot(context, system_database, "contributors"); - attachLazyOrNot(context, system_database, "users"); - attachLazyOrNot(context, system_database, "roles"); - attachLazyOrNot(context, system_database, "grants"); - attachLazyOrNot(context, system_database, "role_grants"); - attachLazyOrNot(context, system_database, "current_roles"); - attachLazyOrNot(context, system_database, "enabled_roles"); - attachLazyOrNot(context, system_database, "settings_profiles"); - attachLazyOrNot(context, system_database, "settings_profile_elements"); - attachLazyOrNot(context, system_database, "row_policies"); - attachLazyOrNot(context, system_database, "quotas"); - attachLazyOrNot(context, system_database, "quota_limits"); - attachLazyOrNot(context, system_database, "quota_usage"); - attachLazyOrNot(context, system_database, "quotas_usage"); - attachLazyOrNot(context, system_database, "user_directories"); - attachLazyOrNot(context, system_database, "privileges"); - attachLazyOrNot(context, system_database, "errors"); - attachLazyOrNot(context, system_database, "warnings"); - attachLazyOrNot(context, system_database, "data_skipping_indices"); - attachLazyOrNot(context, system_database, "licenses"); - attachLazyOrNot(context, system_database, "time_zones"); - attachLazyOrNot(context, system_database, "backups"); - attachLazyOrNot(context, system_database, "schema_inference_cache"); - attachLazyOrNot(context, system_database, "dropped_tables"); - attachLazyOrNot(context, system_database, "scheduler"); + attach(context, system_database, "one"); + attach(context, system_database, "numbers", false); + attach(context, system_database, "numbers_mt", true); + attach(context, system_database, "zeros", false); + attach(context, system_database, "zeros_mt", true); + attach(context, system_database, "databases"); + attach(context, system_database, "tables"); + attach(context, system_database, "columns"); + attach(context, system_database, "functions"); + attach(context, system_database, "events"); + attach(context, system_database, "settings"); + attach(context, system_database, "server_settings"); + attach(context, system_database, "settings_changes"); + attach>(context, system_database, "merge_tree_settings"); + attach>(context, system_database, "replicated_merge_tree_settings"); + attach(context, system_database, "build_options"); + attach(context, system_database, "formats"); + attach(context, system_database, "table_functions"); + attach(context, system_database, "aggregate_function_combinators"); + attach(context, system_database, "data_type_families"); + attach(context, system_database, "collations"); + attach(context, system_database, "table_engines"); + attach(context, system_database, "contributors"); + attach(context, system_database, "users"); + attach(context, system_database, "roles"); + attach(context, system_database, "grants"); + attach(context, system_database, "role_grants"); + attach(context, system_database, "current_roles"); + attach(context, system_database, "enabled_roles"); + attach(context, system_database, "settings_profiles"); + attach(context, system_database, "settings_profile_elements"); + attach(context, system_database, "row_policies"); + attach(context, system_database, "quotas"); + attach(context, system_database, "quota_limits"); + attach(context, system_database, "quota_usage"); + attach(context, system_database, "quotas_usage"); + attach(context, system_database, "user_directories"); + attach(context, system_database, "privileges"); + attach(context, system_database, "errors"); + attach(context, system_database, "warnings"); + attach(context, system_database, "data_skipping_indices"); + attach(context, system_database, "licenses"); + attach(context, system_database, "time_zones"); + attach(context, system_database, "backups"); + attach(context, system_database, "schema_inference_cache"); + attach(context, system_database, "dropped_tables"); + attach(context, system_database, "scheduler"); #if defined(__ELF__) && !defined(OS_FREEBSD) - attachLazyOrNot(context, system_database, "symbols"); + attach(context, system_database, "symbols"); #endif #if USE_RDKAFKA - attachLazyOrNot(context, system_database, "kafka_consumers"); + attach(context, system_database, "kafka_consumers"); #endif #ifdef OS_LINUX - attachLazyOrNot(context, system_database, "stack_trace"); + attach(context, system_database, "stack_trace"); #endif #if USE_ROCKSDB - attachLazyOrNot(context, system_database, "rocksdb"); + attach(context, system_database, "rocksdb"); #endif } -template void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database); -template void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database); void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper) { - attachSystemTablesLocal(context, system_database); + attachSystemTablesLocal(context, system_database); attach(context, system_database, "parts"); attach(context, system_database, "projection_parts"); diff --git a/src/Storages/System/attachSystemTables.h b/src/Storages/System/attachSystemTables.h index d09189ca92f..4c1a79f84dd 100644 --- a/src/Storages/System/attachSystemTables.h +++ b/src/Storages/System/attachSystemTables.h @@ -9,12 +9,8 @@ namespace DB class AsynchronousMetrics; class IDatabase; -template -void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database); void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper); +void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database); void attachSystemTablesAsync(ContextPtr context, IDatabase & system_database, AsynchronousMetrics & async_metrics); -extern template void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database); -extern template void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database); - } diff --git a/src/Storages/System/attachSystemTablesImpl.h b/src/Storages/System/attachSystemTablesImpl.h index 45fd7957513..a1fae985d92 100644 --- a/src/Storages/System/attachSystemTablesImpl.h +++ b/src/Storages/System/attachSystemTablesImpl.h @@ -1,42 +1,15 @@ #pragma once #include -#include #include namespace DB { -template -void attachLazy(ContextPtr context, IDatabase & system_database, const String & table_name, StorageArgs && ... args) -{ - if (system_database.getUUID() == UUIDHelpers::Nil) - { - /// Attach to Ordinary database. - auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name); - system_database.registerLazyTable(context, table_name, [table_id, ... captured_args = std::forward(args)] mutable - { - return std::make_shared(table_id, std::forward(captured_args)...); - }); - } - else - { - /// Attach to Atomic database. - /// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages - /// and path is actually not used - auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4()); - DatabaseCatalog::instance().addUUIDMapping(table_id.uuid); - String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid); - system_database.registerLazyTable(context, table_name, [table_id, ... captured_args = std::forward(args)] mutable - { - return std::make_shared(table_id, std::forward(captured_args)...); - }, path); - } -} - template void attach(ContextPtr context, IDatabase & system_database, const String & table_name, StorageArgs && ... args) { + assert(system_database.getDatabaseName() == DatabaseCatalog::SYSTEM_DATABASE); if (system_database.getUUID() == UUIDHelpers::Nil) { /// Attach to Ordinary database. @@ -55,15 +28,4 @@ void attach(ContextPtr context, IDatabase & system_database, const String & tabl } } -template -void attachLazyOrNot(ContextPtr context, IDatabase & system_database, const String & table_name, StorageArgs && ... args) -{ - assert(system_database.getDatabaseName() == DatabaseCatalog::SYSTEM_DATABASE); - - if constexpr (lazy) - attachLazy(context, system_database, table_name, std::forward(args)...); - else - attach(context, system_database, table_name, std::forward(args)...); -} - }