From d9f4b4d2c05d553d287f4f3551c406fe599b50d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=8C=E6=B6=9B?= Date: Thu, 30 Nov 2023 10:56:31 +0800 Subject: [PATCH] support parts, tables, databases --- programs/server/Server.cpp | 3 + src/Core/ServerSettings.h | 3 + src/Databases/DatabaseLazy.cpp | 7 ++- src/Databases/DatabaseOnDisk.cpp | 1 - src/Databases/DatabasesCommon.cpp | 2 - src/Databases/IDatabase.cpp | 15 +++++ src/Databases/IDatabase.h | 4 +- src/Interpreters/Context.cpp | 34 +++++++++++ src/Interpreters/Context.h | 3 + tests/config/config.d/max_num_to_warn.xml | 5 ++ tests/config/install.sh | 1 + .../02931_max_num_to_warn.reference | 3 + .../0_stateless/02931_max_num_to_warn.sql | 61 +++++++++++++++++++ 13 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 tests/config/config.d/max_num_to_warn.xml create mode 100644 tests/queries/0_stateless/02931_max_num_to_warn.reference create mode 100644 tests/queries/0_stateless/02931_max_num_to_warn.sql diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 11ad06640c8..c3d7bff00d6 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1280,6 +1280,9 @@ try global_context->setMaxTableSizeToDrop(server_settings_.max_table_size_to_drop); global_context->setMaxPartitionSizeToDrop(server_settings_.max_partition_size_to_drop); + global_context->setMaxTableNumToWarn(server_settings_.max_table_num_to_warn); + global_context->setMaxDatabaseNumToWarn(server_settings_.max_database_num_to_warn); + global_context->setMaxPartNumToWarn(server_settings_.max_part_num_to_warn); ConcurrencyControl::SlotCount concurrent_threads_soft_limit = ConcurrencyControl::Unlimited; if (server_settings_.concurrent_threads_soft_limit_num > 0 && server_settings_.concurrent_threads_soft_limit_num < concurrent_threads_soft_limit) diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index eca4b7424de..0a48c73c1fd 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -79,6 +79,9 @@ namespace DB \ M(UInt64, max_table_size_to_drop, 50000000000lu, "If size of a table is greater than this value (in bytes) than table could not be dropped with any DROP query.", 0) \ M(UInt64, max_partition_size_to_drop, 50000000000lu, "Same as max_table_size_to_drop, but for the partitions.", 0) \ + M(UInt64, max_table_num_to_warn, 5000lu, "If number of tables is greater than this value, server will create a warning that will displayed to user.", 0) \ + M(UInt64, max_database_num_to_warn, 1000lu, "If number of databases is greater than this value, server will create a warning that will displayed to user.", 0) \ + M(UInt64, max_part_num_to_warn, 100000lu, "If number of databases is greater than this value, server will create a warning that will displayed to user.", 0) \ M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \ M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ \ diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 9070c0b6ee7..75e4b19d628 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -18,6 +18,10 @@ namespace fs = std::filesystem; +namespace CurrentMetrics { + extern const Metric AttachedTable; +} + namespace DB { @@ -64,7 +68,6 @@ void DatabaseLazy::createTable( SCOPE_EXIT_MEMORY_SAFE({ clearExpiredTables(); }); if (!endsWith(table->getName(), "Log")) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Lazy engine can be used only with *Log tables."); - std::cout<<"======= Flag 1"<second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name); + CurrentMetrics::add(CurrentMetrics::AttachedTable, 1); } StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & table_name) @@ -190,6 +194,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta if (it->second.expiration_iterator != cache_expiration_queue.end()) cache_expiration_queue.erase(it->second.expiration_iterator); tables_cache.erase(it); + CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1); } return res; } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index f1fc6bbdd7a..549711150b8 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -263,7 +263,6 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora try { /// Add a table to the map of known tables. - std::cout<<"===== flag 3"<getStorageID(); if (table_id.database_name != database_name) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed to `{}`, cannot create table in `{}`", diff --git a/src/Databases/IDatabase.cpp b/src/Databases/IDatabase.cpp index 09640d2f86e..b4fd5ea4612 100644 --- a/src/Databases/IDatabase.cpp +++ b/src/Databases/IDatabase.cpp @@ -5,8 +5,14 @@ #include #include #include +#include +namespace CurrentMetrics +{ + extern const Metric AttachedDatabase; +} + namespace DB { @@ -29,6 +35,15 @@ StoragePtr IDatabase::getTable(const String & name, ContextPtr context) const throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} does not exist. Maybe you meant {}?", backQuoteIfNeed(getDatabaseName()), backQuoteIfNeed(name), backQuoteIfNeed(names[0])); } +IDatabase::IDatabase(String database_name_) : database_name(std::move(database_name_)) { + CurrentMetrics::add(CurrentMetrics::AttachedDatabase, 1); +} + +IDatabase::~IDatabase() +{ + CurrentMetrics::sub(CurrentMetrics::AttachedDatabase, 1); +} + std::vector> IDatabase::getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const { /// Cannot backup any table because IDatabase doesn't own any tables. diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index e886f1adae3..7a8f9cb2e2b 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -122,7 +122,7 @@ class IDatabase : public std::enable_shared_from_this { public: IDatabase() = delete; - explicit IDatabase(String database_name_) : database_name(std::move(database_name_)) {} + explicit IDatabase(String database_name_); /// Get name of database engine. virtual String getEngineName() const = 0; @@ -357,7 +357,7 @@ public: /// Creates a table restored from backup. virtual void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr context, std::shared_ptr restore_coordination, UInt64 timeout_ms); - virtual ~IDatabase() = default; + virtual ~IDatabase(); protected: virtual ASTPtr getCreateTableQueryImpl(const String & /*name*/, ContextPtr /*context*/, bool throw_on_error) const diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 185f9782da5..a2c94d59826 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -142,6 +142,9 @@ namespace CurrentMetrics extern const Metric IOWriterThreads; extern const Metric IOWriterThreadsActive; extern const Metric IOWriterThreadsScheduled; + extern const Metric AttachedTable; + extern const Metric AttachedDatabase; + extern const Metric PartsActive; } @@ -323,6 +326,10 @@ struct ContextSharedPart : boost::noncopyable std::optional replicated_merge_tree_settings TSA_GUARDED_BY(mutex); /// Settings of ReplicatedMergeTree* engines. std::atomic_size_t max_table_size_to_drop = 50000000000lu; /// Protects MergeTree tables from accidental DROP (50GB by default) std::atomic_size_t max_partition_size_to_drop = 50000000000lu; /// Protects MergeTree partitions from accidental DROP (50GB by default) + /// std::atomic_size_t max_table_size_to_warn; + std::atomic_size_t max_database_num_to_warn = 1000lu; + std::atomic_size_t max_table_num_to_warn = 5000lu; + std::atomic_size_t max_part_num_to_warn = 100000lu; /// No lock required for format_schema_path modified only during initialization String format_schema_path; /// Path to a directory that contains schema files used by input formats. mutable OnceFlag action_locks_manager_initialized; @@ -829,6 +836,15 @@ Strings Context::getWarnings() const { SharedLockGuard lock(shared->mutex); common_warnings = shared->warnings; + + if (CurrentMetrics::get(CurrentMetrics::AttachedTable) > static_cast(shared->max_table_num_to_warn)) + common_warnings.emplace_back(fmt::format("Attached tables is more than {}", shared->max_table_num_to_warn)); + + if (CurrentMetrics::get(CurrentMetrics::AttachedDatabase) > static_cast(shared->max_database_num_to_warn)) + common_warnings.emplace_back(fmt::format("Attached databases is more than {}", shared->max_table_num_to_warn)); + + if (CurrentMetrics::get(CurrentMetrics::PartsActive) > static_cast(shared->max_part_num_to_warn)) + common_warnings.emplace_back(fmt::format("Active parts is more than {}", shared->max_part_num_to_warn)); } /// Make setting's name ordered std::set obsolete_settings; @@ -3322,6 +3338,24 @@ UInt16 Context::getServerPort(const String & port_name) const return it->second; } +void Context::setMaxPartNumToWarn(size_t max_part_to_warn) +{ + SharedLockGuard lock(shared->mutex); + shared->max_part_num_to_warn = max_part_to_warn; +} + +void Context::setMaxTableNumToWarn(size_t max_table_to_warn) +{ + SharedLockGuard lock(shared->mutex); + shared->max_table_num_to_warn= max_table_to_warn; +} + +void Context::setMaxDatabaseNumToWarn(size_t max_database_to_warn) +{ + SharedLockGuard lock(shared->mutex); + shared->max_database_num_to_warn= max_database_to_warn; +} + std::shared_ptr Context::getCluster(const std::string & cluster_name) const { if (auto res = tryGetCluster(cluster_name)) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 7ae567548dd..517ccded6bd 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -838,6 +838,9 @@ public: void setHTTPHeaderFilter(const Poco::Util::AbstractConfiguration & config); const HTTPHeaderFilter & getHTTPHeaderFilter() const; + void setMaxTableNumToWarn(size_t max_table_to_warn); + void setMaxDatabaseNumToWarn(size_t max_database_to_warn); + void setMaxPartNumToWarn(size_t max_part_to_warn); /// The port that the server listens for executing SQL queries. UInt16 getTCPPort() const; diff --git a/tests/config/config.d/max_num_to_warn.xml b/tests/config/config.d/max_num_to_warn.xml new file mode 100644 index 00000000000..77d68998f8e --- /dev/null +++ b/tests/config/config.d/max_num_to_warn.xml @@ -0,0 +1,5 @@ + + 10 + 10 + 10 + diff --git a/tests/config/install.sh b/tests/config/install.sh index 417a413bbec..9538b59ad9d 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -16,6 +16,7 @@ mkdir -p $DEST_SERVER_PATH/users.d/ mkdir -p $DEST_CLIENT_PATH ln -sf $SRC_PATH/config.d/zookeeper_write.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/max_num_to_warn.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/listen.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/text_log.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/blob_storage_log.xml $DEST_SERVER_PATH/config.d/ diff --git a/tests/queries/0_stateless/02931_max_num_to_warn.reference b/tests/queries/0_stateless/02931_max_num_to_warn.reference new file mode 100644 index 00000000000..76d86352bfc --- /dev/null +++ b/tests/queries/0_stateless/02931_max_num_to_warn.reference @@ -0,0 +1,3 @@ +Attached tables is more than 10 +Attached databases is more than 10 +Active parts is more than 10 diff --git a/tests/queries/0_stateless/02931_max_num_to_warn.sql b/tests/queries/0_stateless/02931_max_num_to_warn.sql new file mode 100644 index 00000000000..cd9a4ebe5fa --- /dev/null +++ b/tests/queries/0_stateless/02931_max_num_to_warn.sql @@ -0,0 +1,61 @@ +CREATE TABLE test_max_num_to_warn_1 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_2 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_3 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_4 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_5 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_6 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_7 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_8 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_9 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_10 (id Int32, str String) Engine=Memory; +CREATE TABLE test_max_num_to_warn_11 (id Int32, str String) Engine=Memory; + +CREATE DATABASE test_max_num_to_warn_1; +CREATE DATABASE test_max_num_to_warn_2; +CREATE DATABASE test_max_num_to_warn_3; +CREATE DATABASE test_max_num_to_warn_4; +CREATE DATABASE test_max_num_to_warn_5; +CREATE DATABASE test_max_num_to_warn_6; +CREATE DATABASE test_max_num_to_warn_7; +CREATE DATABASE test_max_num_to_warn_8; +CREATE DATABASE test_max_num_to_warn_9; +CREATE DATABASE test_max_num_to_warn_10; +CREATE DATABASE test_max_num_to_warn_11; + +INSERT INTO test_max_num_to_warn_1 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_2 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_3 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_4 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_5 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_6 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_7 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_8 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_9 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_10 VALUES (1, 'Hello'); +INSERT INTO test_max_num_to_warn_11 VALUES (1, 'Hello'); + +SELECT * FROM system.warnings where message in ('Attached tables is more than 10', 'Attached databases is more than 10', 'Active parts is more than 10'); + +DROP TABLE test_max_num_to_warn_1; +DROP TABLE test_max_num_to_warn_2; +DROP TABLE test_max_num_to_warn_3; +DROP TABLE test_max_num_to_warn_4; +DROP TABLE test_max_num_to_warn_5; +DROP TABLE test_max_num_to_warn_6; +DROP TABLE test_max_num_to_warn_7; +DROP TABLE test_max_num_to_warn_8; +DROP TABLE test_max_num_to_warn_9; +DROP TABLE test_max_num_to_warn_10; +DROP TABLE test_max_num_to_warn_11; + +DROP DATABASE test_max_num_to_warn_1; +DROP DATABASE test_max_num_to_warn_2; +DROP DATABASE test_max_num_to_warn_3; +DROP DATABASE test_max_num_to_warn_4; +DROP DATABASE test_max_num_to_warn_5; +DROP DATABASE test_max_num_to_warn_6; +DROP DATABASE test_max_num_to_warn_7; +DROP DATABASE test_max_num_to_warn_8; +DROP DATABASE test_max_num_to_warn_9; +DROP DATABASE test_max_num_to_warn_10; +DROP DATABASE test_max_num_to_warn_11;