thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Hard limit on replicated tables, dicts, views

Browse Source
This commit is contained in:
Кирилл Гарбар 2024-10-28 19:00:37 +03:00
parent f41d604f28
commit 2c3363e40e
9 changed files with 121 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Common/CurrentMetrics.cpp
View File

@ -242,6 +242,7 @@
M(PartsActive, "Active data part, used by current and upcoming SELECTs.") \
M(AttachedDatabase, "Active databases.") \
M(AttachedTable, "Active tables.") \
M(AttachedReplicatedTable, "Active replicated tables.") \
M(AttachedView, "Active views.") \
M(AttachedDictionary, "Active dictionaries.") \
M(PartsOutdated, "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.") \

3
src/Core/ServerSettings.cpp
View File

@ -128,7 +128,10 @@ namespace DB
M(UInt64, max_database_num_to_warn, 1000lu, "If the number of databases is greater than this value, the server will create a warning that will displayed to user.", 0) \
M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \
M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
M(UInt64, max_replicated_table_num_to_throw, 0lu, "If number of replicated tables is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \
M(UInt64, max_dictionary_num_to_throw, 0lu, "If number of dictionaries is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
M(UInt64, max_view_num_to_throw, 0lu, "If number of views is greater than this value, server will throw an exception. 0 means no limitation. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
M(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users. Zero means unlimited", 0) \
M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \
M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \

6
src/Databases/DatabasesCommon.cpp
View File

@ -382,7 +382,8 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
if (!table_storage->isSystemStorage() && !DatabaseCatalog::isPredefinedDatabase(database_name))
{
LOG_TEST(log, "Counting detached table {} to database {}", table_name, database_name);
CurrentMetrics::sub(getAttachedCounterForStorage(table_storage));
for (auto metric : getAttachedCountersForStorage(table_storage))
CurrentMetrics::sub(metric);
}
auto table_id = table_storage->getStorageID();
@ -430,7 +431,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
if (!table->isSystemStorage() && !DatabaseCatalog::isPredefinedDatabase(database_name))
{
LOG_TEST(log, "Counting attached table {} to database {}", table_name, database_name);
CurrentMetrics::add(getAttachedCounterForStorage(table));
for (auto metric : getAttachedCountersForStorage(table))
CurrentMetrics::add(metric);
}
}

63
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -98,6 +98,9 @@
namespace CurrentMetrics
{
extern const Metric AttachedTable;
extern const Metric AttachedReplicatedTable;
extern const Metric AttachedDictionary;
extern const Metric AttachedView;
}
namespace DB
@ -146,7 +149,10 @@ namespace ServerSetting
{
extern const ServerSettingsBool ignore_empty_sql_security_in_create_view_query;
extern const ServerSettingsUInt64 max_database_num_to_throw;
extern const ServerSettingsUInt64 max_dictionary_num_to_throw;
extern const ServerSettingsUInt64 max_table_num_to_throw;
extern const ServerSettingsUInt64 max_replicated_table_num_to_throw;
extern const ServerSettingsUInt64 max_view_num_to_throw;
}
namespace ErrorCodes
@ -1914,16 +1920,8 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
}
}
UInt64 table_num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_table_num_to_throw];
if (table_num_limit > 0 && !internal)
{
UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable);
if (table_count >= table_num_limit)
throw Exception(ErrorCodes::TOO_MANY_TABLES,
"Too many tables. "
"The limit (server configuration parameter `max_table_num_to_throw`) is set to {}, the current number of tables is {}",
table_num_limit, table_count);
}
if (!internal)
throwIfTooManyEntities(create, res);
database->createTable(getContext(), create.getTable(), res, query_ptr);
@ -1950,6 +1948,51 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
}
void InterpreterCreateQuery::throwIfTooManyEntities(ASTCreateQuery & create, StoragePtr storage) const
{
if (auto * replicated_storage = typeid_cast<StorageReplicatedMergeTree *>(storage.get()))
{
UInt64 num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_replicated_table_num_to_throw];
UInt64 attached_count = CurrentMetrics::get(CurrentMetrics::AttachedReplicatedTable);
if (attached_count >= num_limit)
throw Exception(ErrorCodes::TOO_MANY_TABLES,
"Too many replicated tables. "
"The limit (server configuration parameter `max_replicated_table_num_to_throw`) is set to {}, the current number is {}",
num_limit, attached_count);
}
else if (create.is_dictionary)
{
UInt64 num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_dictionary_num_to_throw];
UInt64 attached_count = CurrentMetrics::get(CurrentMetrics::AttachedDictionary);
if (attached_count >= num_limit)
throw Exception(ErrorCodes::TOO_MANY_TABLES,
"Too many dictionaries. "
"The limit (server configuration parameter `max_dictionary_num_to_throw`) is set to {}, the current number is {}",
num_limit, attached_count);
}
else if (create.isView())
{
UInt64 num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_view_num_to_throw];
UInt64 attached_count = CurrentMetrics::get(CurrentMetrics::AttachedView);
if (attached_count >= num_limit)
throw Exception(ErrorCodes::TOO_MANY_TABLES,
"Too many views. "
"The limit (server configuration parameter `max_view_num_to_throw`) is set to {}, the current number is {}",
num_limit, attached_count);
}
else
{
UInt64 num_limit = getContext()->getGlobalContext()->getServerSettings()[ServerSetting::max_table_num_to_throw];
UInt64 attached_count = CurrentMetrics::get(CurrentMetrics::AttachedTable);
if (attached_count >= num_limit)
throw Exception(ErrorCodes::TOO_MANY_TABLES,
"Too many tables. "
"The limit (server configuration parameter `max_table_num_to_throw`) is set to {}, the current number is {}",
num_limit, attached_count);
}
}
BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create,
const InterpreterCreateQuery::TableProperties & properties, LoadingStrictnessLevel mode)
{

2
src/Interpreters/InterpreterCreateQuery.h
View File

@ -122,6 +122,8 @@ private:
BlockIO executeQueryOnCluster(ASTCreateQuery & create);
void throwIfTooManyEntities(ASTCreateQuery & create, StoragePtr storage) const;
ASTPtr query_ptr;
/// Skip safety threshold when loading tables.

16
src/Storages/Utils.cpp
View File

@ -1,10 +1,13 @@
#include <vector>
#include <Storages/Utils.h>
#include <Storages/IStorage.h>
#include <Storages/StorageReplicatedMergeTree.h>
namespace CurrentMetrics
{
extern const Metric AttachedTable;
extern const Metric AttachedReplicatedTable;
extern const Metric AttachedView;
extern const Metric AttachedDictionary;
}
@ -12,17 +15,20 @@ namespace CurrentMetrics
namespace DB
{
CurrentMetrics::Metric getAttachedCounterForStorage(const StoragePtr & storage)
std::vector<CurrentMetrics::Metric> getAttachedCountersForStorage(const StoragePtr & storage)
{
if (storage->isView())
{
return CurrentMetrics::AttachedView;
return {CurrentMetrics::AttachedView};
}
if (storage->isDictionary())
{
return CurrentMetrics::AttachedDictionary;
return {CurrentMetrics::AttachedDictionary};
}
return CurrentMetrics::AttachedTable;
if (auto * replicated_storage = typeid_cast<StorageReplicatedMergeTree *>(storage.get()))
{
return {CurrentMetrics::AttachedTable, CurrentMetrics::AttachedReplicatedTable};
}
return {CurrentMetrics::AttachedTable};
}
}

2
src/Storages/Utils.h
View File

@ -6,5 +6,5 @@
namespace DB
{
CurrentMetrics::Metric getAttachedCounterForStorage(const StoragePtr & storage);
std::vector<CurrentMetrics::Metric> getAttachedCountersForStorage(const StoragePtr & storage);
}

12
tests/integration/test_table_db_num_limit/config/config.xml
View File

@ -1,5 +1,17 @@
<clickhouse>
<remote_servers>
<cluster>
<shard>
<replica>
<host>node1</host>
<port>9000</port>
</replica>
</shard>
</cluster>
</remote_servers>
<max_table_num_to_throw>10</max_table_num_to_throw>
<max_replicated_table_num_to_throw>5</max_replicated_table_num_to_throw>
<max_database_num_to_throw>10</max_database_num_to_throw>
</clickhouse>

43
tests/integration/test_table_db_num_limit/test.py
View File

@ -1,11 +1,14 @@
import pytest
from helpers.client import QueryRuntimeException
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
node = cluster.add_instance("node", main_configs=["config/config.xml"])
node = cluster.add_instance(
"node1",
with_zookeeper=True,
main_configs=["config/config.xml"],
)
@pytest.fixture(scope="module")
@ -24,10 +27,9 @@ def test_table_db_limit(started_cluster):
for i in range(9):
node.query("create database db{}".format(i))
with pytest.raises(QueryRuntimeException) as exp_info:
node.query("create database db_exp".format(i))
assert "TOO_MANY_DATABASES" in str(exp_info)
assert "TOO_MANY_DATABASES" in node.query_and_get_error(
"create database db_exp".format(i)
)
for i in range(10):
node.query("create table t{} (a Int32) Engine = Log".format(i))
@ -35,13 +37,36 @@ def test_table_db_limit(started_cluster):
# This checks that system tables are not accounted in the number of tables.
node.query("system flush logs")
# Regular tables
for i in range(10):
node.query("drop table t{}".format(i))
for i in range(10):
node.query("create table t{} (a Int32) Engine = Log".format(i))
with pytest.raises(QueryRuntimeException) as exp_info:
node.query("create table default.tx (a Int32) Engine = Log")
assert "TOO_MANY_TABLES" in node.query_and_get_error(
"create table default.tx (a Int32) Engine = Log"
)
assert "TOO_MANY_TABLES" in str(exp_info)
# Replicated tables
for i in range(10):
node.query("drop table t{}".format(i))
for i in range(5):
node.query(
"create table t{} (a Int32) Engine = ReplicatedMergeTree('/clickhouse/tables/t{}', 'r1') order by a".format(
i, i
)
)
assert "Too many replicated tables" in node.query_and_get_error(
"create table tx (a Int32) Engine = ReplicatedMergeTree('/clickhouse/tables/tx', 'r1') order by a"
)
# Checks that replicated tables are also counted as regular tables
for i in range(5, 10):
node.query("create table t{} (a Int32) Engine = Log".format(i))
assert "TOO_MANY_TABLES" in node.query_and_get_error(
"create table tx (a Int32) Engine = Log"
)