diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 16e1cfd0b69..09b53eb8c64 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -370,7 +370,7 @@ class IColumn; \ M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ M(Int64, postgresql_connection_pool_wait_timeout, -1, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ - M(UInt64, storage_external_distributed_max_addresses, 1000, "Maximum number of addresses for storage ExternalDistributed.", 0) \ + M(UInt64, glob_expansion_max_elements, 1000, "Maximum number of allowed addresses (For external storages, table functions, etc).", 0) \ \ M(Seconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \ M(UInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up an incredible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \ diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 4ab41044683..40d4f805dd4 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -145,7 +145,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { auto mysql_database_settings = std::make_unique(); /// Split into replicas if needed. - size_t max_addresses = context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = context.getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); auto mysql_pool = mysqlxx::PoolWithFailover(mysql_database_name, addresses, mysql_user_name, mysql_user_password); @@ -249,7 +249,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String use_table_cache = safeGetLiteralValue(engine_args[4], engine_name); /// Split into replicas if needed. - size_t max_addresses = context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = context.getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 5432); /// no connection is made here diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 9197f2be9ec..d371f01d9e8 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -119,7 +119,7 @@ std::string MySQLDictionarySource::getUpdateFieldAndDate() } } -BlockInputStreamPtr MySQLDictionarySource::loadBase(const String & query) +BlockInputStreamPtr MySQLDictionarySource::loadFromQuery(const String & query) { return std::make_shared( pool, query, sample_block, max_block_size, close_connection, false, max_tries_for_mysql_block_input_stream); @@ -131,7 +131,7 @@ BlockInputStreamPtr MySQLDictionarySource::loadAll() last_modification = getLastModification(connection, false); LOG_TRACE(log, load_all_query); - return loadBase(load_all_query); + return loadFromQuery(load_all_query); } BlockInputStreamPtr MySQLDictionarySource::loadUpdatedAll() @@ -141,21 +141,21 @@ BlockInputStreamPtr MySQLDictionarySource::loadUpdatedAll() std::string load_update_query = getUpdateFieldAndDate(); LOG_TRACE(log, load_update_query); - return loadBase(load_update_query); + return loadFromQuery(load_update_query); } BlockInputStreamPtr MySQLDictionarySource::loadIds(const std::vector & ids) { /// We do not log in here and do not update the modification time, as the request can be large, and often called. const auto query = query_builder.composeLoadIdsQuery(ids); - return loadBase(query); + return loadFromQuery(query); } BlockInputStreamPtr MySQLDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { /// We do not log in here and do not update the modification time, as the request can be large, and often called. const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN); - return loadBase(query); + return loadFromQuery(query); } bool MySQLDictionarySource::isModified() const diff --git a/src/Dictionaries/MySQLDictionarySource.h b/src/Dictionaries/MySQLDictionarySource.h index b1b139159f9..dcd911146aa 100644 --- a/src/Dictionaries/MySQLDictionarySource.h +++ b/src/Dictionaries/MySQLDictionarySource.h @@ -49,8 +49,6 @@ public: BlockInputStreamPtr loadKeys(const Columns & key_columns, const std::vector & requested_rows) override; - BlockInputStreamPtr loadBase(const String & query); - bool isModified() const override; bool supportsSelectiveLoad() const override; @@ -62,6 +60,8 @@ public: std::string toString() const override; private: + BlockInputStreamPtr loadFromQuery(const String & query); + std::string getUpdateFieldAndDate(); static std::string quoteForLike(const std::string s); diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 0f84724b343..a8e02236295 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -44,7 +44,7 @@ StorageExternalDistributed::StorageExternalDistributed( storage_metadata.setConstraints(constraints_); setInMemoryMetadata(storage_metadata); - size_t max_addresses = context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = context.getSettingsRef().glob_expansion_max_elements; std::vector shards_descriptions = parseRemoteDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); std::vector> addresses; diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index c01cf03d644..66f6b374339 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -236,7 +236,7 @@ void registerStorageMySQL(StorageFactory & factory) const String & remote_table = engine_args[2]->as().value.safeGet(); const String & username = engine_args[3]->as().value.safeGet(); const String & password = engine_args[4]->as().value.safeGet(); - size_t max_addresses = args.context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = args.context.getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); mysqlxx::PoolWithFailover pool(remote_database, addresses, username, password); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 2361ce0c96f..d630cb26342 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -308,7 +308,7 @@ void registerStoragePostgreSQL(StorageFactory & factory) auto host_port = engine_args[0]->as().value.safeGet(); /// Split into replicas if needed. - size_t max_addresses = args.context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = args.context.getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 5432); const String & remote_database = engine_args[1]->as().value.safeGet(); diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 0bdd934b64d..d29cb8d55c2 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -61,7 +61,7 @@ void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, const Conte password = args[4]->as().value.safeGet(); /// Split into replicas if needed. 3306 is the default MySQL port number - size_t max_addresses = context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = context.getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); pool.emplace(remote_database_name, addresses, user_name, password); diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index eb3ae2716e8..e1ee6df1ba1 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -70,7 +70,7 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const /// Split into replicas if needed. 5432 is a default postgresql port. const auto & host_port = args[0]->as().value.safeGet(); - size_t max_addresses = context.getSettingsRef().storage_external_distributed_max_addresses; + size_t max_addresses = context.getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 5432); remote_table_name = args[2]->as().value.safeGet();