diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 36f774a3c12..df1e090457a 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,6 @@ #include #include #include -#include /** A tool for evaluating ClickHouse performance. @@ -58,8 +58,9 @@ static constexpr std::string_view DEFAULT_CLIENT_NAME = "benchmark"; namespace ErrorCodes { - extern const int CANNOT_BLOCK_SIGNAL; - extern const int EMPTY_DATA_PASSED; +extern const int BAD_ARGUMENTS; +extern const int CANNOT_BLOCK_SIGNAL; +extern const int EMPTY_DATA_PASSED; } class Benchmark : public Poco::Util::Application @@ -637,7 +638,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ; Settings settings; - addProgramOptions(settings, desc); + settings.addToProgramOptions(desc); boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 25c94c56aa6..494c761ecab 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -56,6 +56,12 @@ using namespace std::literals; namespace DB { +namespace Setting +{ + extern const SettingsDialect dialect; + extern const SettingsBool use_client_time_zone; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -307,9 +313,9 @@ void Client::initialize(Poco::Util::Application & self) config().setString("password", env_password); /// settings and limits could be specified in config file, but passed settings has higher priority - for (const auto & setting : global_context->getSettingsRef().allUnchanged()) + for (const auto & setting : global_context->getSettingsRef().getUnchangedNames()) { - const auto & name = setting.getName(); + String name{setting}; if (config().has(name)) global_context->setSetting(name, config().getString(name)); } @@ -525,7 +531,7 @@ void Client::connect() } } - if (!client_context->getSettingsRef().use_client_time_zone) + if (!client_context->getSettingsRef()[Setting::use_client_time_zone]) { const auto & time_zone = connection->getServerTimezone(connection_parameters.timeouts); if (!time_zone.empty()) @@ -730,7 +736,7 @@ bool Client::processWithFuzzing(const String & full_query) } // Kusto is not a subject for fuzzing (yet) - if (client_context->getSettingsRef().dialect == DB::Dialect::kusto) + if (client_context->getSettingsRef()[Setting::dialect] == DB::Dialect::kusto) { return true; } @@ -1073,17 +1079,7 @@ void Client::processOptions(const OptionsDescription & options_description, /// Copy settings-related program options to config. /// TODO: Is this code necessary? - for (const auto & setting : global_context->getSettingsRef().all()) - { - const auto & name = setting.getName(); - if (options.count(name)) - { - if (allow_repeated_settings) - config().setString(name, options[name].as().back()); - else - config().setString(name, options[name].as()); - } - } + global_context->getSettingsRef().addToClientOptions(config(), options, allow_repeated_settings); if (options.count("config-file") && options.count("config")) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Two or more configuration files referenced in arguments"); diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index f07387bd395..522b9a74cff 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -3,11 +3,12 @@ #include #include -#include +#include #include #include #include #include +#include #include #include #include @@ -17,7 +18,6 @@ #include #include #include -#include #include #include @@ -35,6 +35,15 @@ #include #include +namespace DB +{ +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} +} namespace DB::ErrorCodes { @@ -99,12 +108,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ; Settings cmd_settings; - for (const auto & field : cmd_settings.all()) - { - std::string_view name = field.getName(); - if (name == "max_parser_depth" || name == "max_query_size") - addProgramOption(cmd_settings, desc, name, field); - } + cmd_settings.addToProgramOptions("max_parser_depth", desc); + cmd_settings.addToProgramOptions("max_query_size", desc); boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); @@ -240,7 +245,14 @@ int mainEntryClickHouseFormat(int argc, char ** argv) size_t approx_query_length = multiple ? find_first_symbols<';'>(pos, end) - pos : end - pos; ASTPtr res = parseQueryAndMovePosition( - parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth, cmd_settings.max_parser_backtracks); + parser, + pos, + end, + "query", + multiple, + cmd_settings[Setting::max_query_size], + cmd_settings[Setting::max_parser_depth], + cmd_settings[Setting::max_parser_backtracks]); std::unique_ptr insert_query_payload; /// If the query is INSERT ... VALUES, then we will try to parse the data. diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 184f147a86a..f678194849d 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -71,6 +71,11 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool allow_introspection_functions; + extern const SettingsLocalFSReadMethod storage_file_read_method; +} namespace ErrorCodes { @@ -83,8 +88,8 @@ void applySettingsOverridesForLocal(ContextMutablePtr context) { Settings settings = context->getSettingsCopy(); - settings.allow_introspection_functions = true; - settings.storage_file_read_method = LocalFSReadMethod::mmap; + settings[Setting::allow_introspection_functions] = true; + settings[Setting::storage_file_read_method] = LocalFSReadMethod::mmap; context->setSettings(settings); } diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index b18a7a90961..7e92e92d345 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -4,7 +4,6 @@ #include #include -#include #include #include #include diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 6c149d7a08c..06dd25ce274 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -2,6 +2,7 @@ #if USE_ODBC +#include #include #include #include @@ -27,6 +28,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 odbc_bridge_connection_pool_size; +} namespace ErrorCodes { @@ -129,8 +134,7 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ const bool external_table_functions_use_nulls = Poco::NumberParser::parseBool(params.get("external_table_functions_use_nulls", "false")); auto connection_holder = ODBCPooledConnectionFactory::instance().get( - validateODBCConnectionString(connection_string), - getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + validateODBCConnectionString(connection_string), getContext()->getSettingsRef()[Setting::odbc_bridge_connection_pool_size]); /// In XDBC tables it is allowed to pass either database_name or schema_name in table definion, but not both of them. /// They both are passed as 'schema' parameter in request URL, so it is not clear whether it is database_name or schema_name passed. diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index 778e6019be1..7b5b69cff63 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -19,6 +19,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 odbc_bridge_connection_pool_size; +} + void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & /*write_event*/) { HTMLForm params(getContext()->getSettingsRef(), request, request.getStream()); @@ -68,7 +73,7 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ nanodbc::ConnectionHolderPtr connection; if (use_connection_pooling) connection = ODBCPooledConnectionFactory::instance().get( - validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + validateODBCConnectionString(connection_string), getContext()->getSettingsRef()[Setting::odbc_bridge_connection_pool_size]); else connection = std::make_shared(validateODBCConnectionString(connection_string)); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 160add9d2ed..04c02288858 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -1,27 +1,28 @@ #include "MainHandler.h" -#include "validateODBCConnectionString.h" -#include "ODBCSource.h" -#include "ODBCSink.h" -#include "getIdentifierQuote.h" +#include #include #include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include -#include +#include "ODBCSink.h" +#include "ODBCSource.h" #include "config.h" +#include "getIdentifierQuote.h" +#include "validateODBCConnectionString.h" #include #include @@ -29,6 +30,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 odbc_bridge_connection_pool_size; +} namespace { @@ -139,7 +144,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse nanodbc::ConnectionHolderPtr connection_handler; if (use_connection_pooling) connection_handler = ODBCPooledConnectionFactory::instance().get( - validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + validateODBCConnectionString(connection_string), getContext()->getSettingsRef()[Setting::odbc_bridge_connection_pool_size]); else connection_handler = std::make_shared(validateODBCConnectionString(connection_string)); diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index e17d7c5b880..c986e61e715 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 odbc_bridge_connection_pool_size; +} + namespace { bool isSchemaAllowed(nanodbc::ConnectionHolderPtr connection_holder) @@ -82,7 +87,7 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer if (use_connection_pooling) connection = ODBCPooledConnectionFactory::instance().get( - validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size); + validateODBCConnectionString(connection_string), getContext()->getSettingsRef()[Setting::odbc_bridge_connection_pool_size]); else connection = std::make_shared(validateODBCConnectionString(connection_string)); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c69d822e383..f0c9719051f 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -148,6 +148,18 @@ /// A minimal file used when the server is run without installation INCBIN(resource_embedded_xml, SOURCE_DIR "/programs/server/embedded.xml"); +namespace DB +{ +namespace Setting +{ + extern const SettingsSeconds http_receive_timeout; + extern const SettingsSeconds http_send_timeout; + extern const SettingsSeconds receive_timeout; + extern const SettingsSeconds send_timeout; +} + +} + namespace CurrentMetrics { extern const Metric Revision; @@ -1831,10 +1843,13 @@ try "Keeper (tcp): " + address.toString(), std::make_unique( new KeeperTCPHandlerFactory( - config_getter, global_context->getKeeperDispatcher(), - global_context->getSettingsRef().receive_timeout.totalSeconds(), - global_context->getSettingsRef().send_timeout.totalSeconds(), - false), server_pool, socket)); + config_getter, + global_context->getKeeperDispatcher(), + global_context->getSettingsRef()[Setting::receive_timeout].totalSeconds(), + global_context->getSettingsRef()[Setting::send_timeout].totalSeconds(), + false), + server_pool, + socket)); }); const char * secure_port_name = "keeper_server.tcp_port_secure"; @@ -1854,9 +1869,13 @@ try "Keeper with secure protocol (tcp_secure): " + address.toString(), std::make_unique( new KeeperTCPHandlerFactory( - config_getter, global_context->getKeeperDispatcher(), - global_context->getSettingsRef().receive_timeout.totalSeconds(), - global_context->getSettingsRef().send_timeout.totalSeconds(), true), server_pool, socket)); + config_getter, + global_context->getKeeperDispatcher(), + global_context->getSettingsRef()[Setting::receive_timeout].totalSeconds(), + global_context->getSettingsRef()[Setting::send_timeout].totalSeconds(), + true), + server_pool, + socket)); #else UNUSED(port); throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for TCP protocol is disabled because Poco library was built without NetSSL support."); @@ -2430,7 +2449,7 @@ void Server::createServers( const Settings & settings = global_context->getSettingsRef(); Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; - http_params->setTimeout(settings.http_receive_timeout); + http_params->setTimeout(settings[Setting::http_receive_timeout]); http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout); http_params->setMaxKeepAliveRequests(static_cast(global_context->getServerSettings().max_keep_alive_requests)); @@ -2469,8 +2488,8 @@ void Server::createServers( { Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, host, port, is_secure); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); + socket.setReceiveTimeout(settings[Setting::receive_timeout]); + socket.setSendTimeout(settings[Setting::send_timeout]); return ProtocolServerAdapter( host, @@ -2497,8 +2516,8 @@ void Server::createServers( { Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); + socket.setReceiveTimeout(settings[Setting::http_receive_timeout]); + socket.setSendTimeout(settings[Setting::http_send_timeout]); return ProtocolServerAdapter( listen_host, @@ -2518,8 +2537,8 @@ void Server::createServers( #if USE_SSL Poco::Net::SecureServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); + socket.setReceiveTimeout(settings[Setting::http_receive_timeout]); + socket.setSendTimeout(settings[Setting::http_send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2541,8 +2560,8 @@ void Server::createServers( { Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); + socket.setReceiveTimeout(settings[Setting::receive_timeout]); + socket.setSendTimeout(settings[Setting::send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2563,8 +2582,8 @@ void Server::createServers( { Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); + socket.setReceiveTimeout(settings[Setting::receive_timeout]); + socket.setSendTimeout(settings[Setting::send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2586,8 +2605,8 @@ void Server::createServers( #if USE_SSL Poco::Net::SecureServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); + socket.setReceiveTimeout(settings[Setting::receive_timeout]); + socket.setSendTimeout(settings[Setting::send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2612,7 +2631,7 @@ void Server::createServers( Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); - socket.setSendTimeout(settings.send_timeout); + socket.setSendTimeout(settings[Setting::send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2629,7 +2648,7 @@ void Server::createServers( Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true); socket.setReceiveTimeout(Poco::Timespan()); - socket.setSendTimeout(settings.send_timeout); + socket.setSendTimeout(settings[Setting::send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2661,8 +2680,8 @@ void Server::createServers( { Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, listen_host, port); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); + socket.setReceiveTimeout(settings[Setting::http_receive_timeout]); + socket.setSendTimeout(settings[Setting::http_send_timeout]); return ProtocolServerAdapter( listen_host, port_name, @@ -2687,7 +2706,7 @@ void Server::createInterserverServers( const Settings & settings = global_context->getSettingsRef(); Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; - http_params->setTimeout(settings.http_receive_timeout); + http_params->setTimeout(settings[Setting::http_receive_timeout]); http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout); /// Now iterate over interserver_listen_hosts @@ -2703,8 +2722,8 @@ void Server::createInterserverServers( { Poco::Net::ServerSocket socket; auto address = socketBindListen(config, socket, interserver_listen_host, port); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); + socket.setReceiveTimeout(settings[Setting::http_receive_timeout]); + socket.setSendTimeout(settings[Setting::http_send_timeout]); return ProtocolServerAdapter( interserver_listen_host, port_name, @@ -2728,8 +2747,8 @@ void Server::createInterserverServers( #if USE_SSL Poco::Net::SecureServerSocket socket; auto address = socketBindListen(config, socket, interserver_listen_host, port, /* secure = */ true); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); + socket.setReceiveTimeout(settings[Setting::http_receive_timeout]); + socket.setSendTimeout(settings[Setting::http_send_timeout]); return ProtocolServerAdapter( interserver_listen_host, port_name, diff --git a/src/Access/ContextAccessParams.cpp b/src/Access/ContextAccessParams.cpp index 07374e9bbf5..fba6eaf9f03 100644 --- a/src/Access/ContextAccessParams.cpp +++ b/src/Access/ContextAccessParams.cpp @@ -1,10 +1,17 @@ #include #include +#include #include namespace DB { +namespace Setting +{ + extern const SettingsBool allow_ddl; + extern const SettingsBool allow_introspection_functions; + extern const SettingsUInt64 readonly; +} ContextAccessParams::ContextAccessParams( std::optional user_id_, @@ -18,9 +25,9 @@ ContextAccessParams::ContextAccessParams( , full_access(full_access_) , use_default_roles(use_default_roles_) , current_roles(current_roles_) - , readonly(settings_.readonly) - , allow_ddl(settings_.allow_ddl) - , allow_introspection(settings_.allow_introspection_functions) + , readonly(settings_[Setting::readonly]) + , allow_ddl(settings_[Setting::allow_ddl]) + , allow_introspection(settings_[Setting::allow_introspection_functions]) , current_database(current_database_) , interface(client_info_.interface) , http_method(client_info_.http_method) diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index a274f6b54f2..483dca7e13f 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -10,10 +10,15 @@ #include #include #include -#include namespace DB { +namespace Setting +{ + extern const SettingsBool allow_ddl; + extern const SettingsUInt64 readonly; +} + namespace ErrorCodes { extern const int READONLY; @@ -164,7 +169,7 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting if (element.writability) new_value = *element.writability; - auto setting_name = Settings::Traits::resolveName(element.setting_name); + auto setting_name = Settings::resolveName(element.setting_name); auto it = constraints.find(setting_name); if (it != constraints.end()) old_value = it->second.writability; @@ -255,7 +260,7 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, ReactionOnViolation reaction, SettingSource source) const { - std::string_view setting_name = Settings::Traits::resolveName(change.name); + std::string_view setting_name = Settings::resolveName(change.name); if (setting_name == "profile") return true; @@ -393,7 +398,7 @@ std::string_view SettingsConstraints::resolveSettingNameWithCache(std::string_vi SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & current_settings, std::string_view setting_name) const { auto resolved_name = resolveSettingNameWithCache(setting_name); - if (!current_settings.allow_ddl && resolved_name == "allow_ddl") + if (!current_settings[Setting::allow_ddl] && resolved_name == "allow_ddl") return Checker(PreformattedMessage::create("Cannot modify 'allow_ddl' setting when DDL queries are prohibited for the user"), ErrorCodes::QUERY_IS_PROHIBITED); @@ -403,11 +408,11 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu * 2 - only read requests, as well as changing settings, except for the `readonly` setting. */ - if (current_settings.readonly > 1 && resolved_name == "readonly") + if (current_settings[Setting::readonly] > 1 && resolved_name == "readonly") return Checker(PreformattedMessage::create("Cannot modify 'readonly' setting in readonly mode"), ErrorCodes::READONLY); auto it = constraints.find(resolved_name); - if (current_settings.readonly == 1) + if (current_settings[Setting::readonly] == 1) { if (it == constraints.end() || it->second.writability != SettingConstraintWritability::CHANGEABLE_IN_READONLY) return Checker(PreformattedMessage::create("Cannot modify '{}' setting in readonly mode", setting_name), @@ -416,9 +421,9 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu else // For both readonly=0 and readonly=2 { if (it == constraints.end()) - return Checker(Settings::Traits::resolveName); // Allowed + return Checker(Settings::resolveName); // Allowed } - return Checker(it->second, Settings::Traits::resolveName); + return Checker(it->second, Settings::resolveName); } SettingsConstraints::Checker SettingsConstraints::getMergeTreeChecker(std::string_view short_name) const diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 9358391cb93..156ffcf197d 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -249,7 +249,7 @@ bool SettingsProfileElements::isBackupAllowed() const bool SettingsProfileElements::isAllowBackupSetting(const String & setting_name) { static constexpr std::string_view ALLOW_BACKUP_SETTING_NAME = "allow_backup"; - return Settings::Traits::resolveName(setting_name) == ALLOW_BACKUP_SETTING_NAME; + return Settings::resolveName(setting_name) == ALLOW_BACKUP_SETTING_NAME; } } diff --git a/src/Access/examples/CMakeLists.txt b/src/Access/examples/CMakeLists.txt index 99d040cec68..18f40c2ef14 100644 --- a/src/Access/examples/CMakeLists.txt +++ b/src/Access/examples/CMakeLists.txt @@ -1,4 +1,4 @@ if (TARGET ch_contrib::krb5) clickhouse_add_executable (kerberos_init kerberos_init.cpp) - target_link_libraries (kerberos_init PRIVATE dbms ch_contrib::krb5) + target_link_libraries (kerberos_init PRIVATE dbms clickhouse_functions ch_contrib::krb5) endif() diff --git a/src/Access/resolveSetting.h b/src/Access/resolveSetting.h index bed7608b118..d9f64a01ff9 100644 --- a/src/Access/resolveSetting.h +++ b/src/Access/resolveSetting.h @@ -46,14 +46,6 @@ inline Field settingCastValueUtil(std::string_view full_name, const Field & valu }); } -inline String settingValueToStringUtil(std::string_view full_name, const Field & value) -{ - return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) - { - return T::valueToStringUtil(short_name, value); - }); -} - inline Field settingStringToValueUtil(std::string_view full_name, const String & str) { return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) @@ -89,10 +81,9 @@ inline String settingFullName(std::string_view short_name) inline std::string resolveSettingName(std::string_view full_name) { - return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) - { - return settingFullName(T::Traits::resolveName(short_name)); - }); + return resolveSetting( + full_name, + [&](std::string_view short_name, SettingsType) { return settingFullName(T::resolveName(short_name)); }); } } diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 082fa11ca8a..bf2a4b18afd 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -14,6 +14,10 @@ static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000; namespace DB { struct Settings; +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -199,7 +203,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( found = *opt; out_properties = found.properties; - if (query_context && query_context->getSettingsRef().log_queries) + if (query_context && query_context->getSettingsRef()[Setting::log_queries]) query_context->addQueryFactoriesInfo( Context::QueryLogFactories::AggregateFunction, is_case_insensitive ? case_insensitive_name : name); @@ -224,7 +228,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( "Aggregate function combinator '{}' is only for internal usage", combinator_name); - if (query_context && query_context->getSettingsRef().log_queries) + if (query_context && query_context->getSettingsRef()[Setting::log_queries]) query_context->addQueryFactoriesInfo(Context::QueryLogFactories::AggregateFunctionCombinator, combinator_name); String nested_name = name.substr(0, name.size() - combinator_name.size()); diff --git a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp index 204edd83215..7fc7251497f 100644 --- a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp +++ b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp @@ -17,6 +17,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_introspection_functions; +} + + namespace ErrorCodes { extern const int FUNCTION_NOT_ALLOWED; @@ -628,7 +634,7 @@ static void check(const std::string & name, const DataTypes & argument_types, co AggregateFunctionPtr createAggregateFunctionFlameGraph(const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings) { - if (!settings->allow_introspection_functions) + if (!(*settings)[Setting::allow_introspection_functions]) throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0"); diff --git a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp index b0240225138..94dff52ade3 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp +++ b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp @@ -18,15 +18,19 @@ #include #include +#include #include namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_funnel_functions; +} constexpr size_t max_events_size = 64; - constexpr size_t min_required_args = 3; namespace ErrorCodes @@ -448,7 +452,7 @@ inline AggregateFunctionPtr createAggregateFunctionSequenceNodeImpl( AggregateFunctionPtr createAggregateFunctionSequenceNode(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings) { - if (settings == nullptr || !settings->allow_experimental_funnel_functions) + if (settings == nullptr || !(*settings)[Setting::allow_experimental_funnel_functions]) { throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, "Aggregate function {} is experimental. " "Set `allow_experimental_funnel_functions` setting to enable it", name); diff --git a/src/AggregateFunctions/AggregateFunctionUniq.cpp b/src/AggregateFunctions/AggregateFunctionUniq.cpp index 748a232641e..e5eec16fa49 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.cpp +++ b/src/AggregateFunctions/AggregateFunctionUniq.cpp @@ -15,7 +15,10 @@ namespace DB { -struct Settings; +namespace Setting +{ + extern const SettingsMaxThreads max_threads; +} namespace ErrorCodes { @@ -149,7 +152,7 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory) auto assign_bool_param = [](const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings) { /// Using two level hash set if we wouldn't be able to merge in parallel can cause ~10% slowdown. - if (settings && settings->max_threads > 1) + if (settings && (*settings)[Setting::max_threads] > 1) return createAggregateFunctionUniq< true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactDataForVariadic, true /* is_able_to_parallelize_merge */>(name, argument_types, params, settings); else diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index f15d067a302..809d8b7347c 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/AggregateFunctions/examples/CMakeLists.txt b/src/AggregateFunctions/examples/CMakeLists.txt index a9033fd8508..8ded1e5472a 100644 --- a/src/AggregateFunctions/examples/CMakeLists.txt +++ b/src/AggregateFunctions/examples/CMakeLists.txt @@ -1,5 +1,5 @@ clickhouse_add_executable (quantile-t-digest quantile-t-digest.cpp) -target_link_libraries (quantile-t-digest PRIVATE dbms clickhouse_aggregate_functions) +target_link_libraries (quantile-t-digest PRIVATE dbms clickhouse_functions clickhouse_aggregate_functions) clickhouse_add_executable (group_array_sorted group_array_sorted.cpp) -target_link_libraries (group_array_sorted PRIVATE dbms clickhouse_aggregate_functions) +target_link_libraries (group_array_sorted PRIVATE dbms clickhouse_functions clickhouse_aggregate_functions) diff --git a/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp b/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp index b68635ab953..5c648402851 100644 --- a/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp @@ -14,6 +14,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_aggregators_of_group_by_keys; +} namespace ErrorCodes { @@ -34,7 +38,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_aggregators_of_group_by_keys) + if (!getSettings()[Setting::optimize_aggregators_of_group_by_keys]) return; /// Collect group by keys. @@ -79,7 +83,7 @@ public: /// Now we visit all nodes in QueryNode, we should remove group_by_keys from stack. void leaveImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_aggregators_of_group_by_keys) + if (!getSettings()[Setting::optimize_aggregators_of_group_by_keys]) return; if (node->getNodeType() == QueryTreeNodeType::FUNCTION) diff --git a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp index 6578a6c5bd7..1c3b7aef673 100644 --- a/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp +++ b/src/Analyzer/Passes/AggregateFunctionsArithmericOperationsPass.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_arithmetic_operations_in_aggregate_functions; +} + namespace ErrorCodes { @@ -56,7 +61,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_arithmetic_operations_in_aggregate_functions) + if (!getSettings()[Setting::optimize_arithmetic_operations_in_aggregate_functions]) return; auto * aggregate_function_node = node->as(); diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp index bf4c1232d79..2744ed493f6 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp @@ -15,6 +15,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_rewrite_array_exists_to_has; +} namespace { @@ -27,7 +31,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_rewrite_array_exists_to_has) + if (!getSettings()[Setting::optimize_rewrite_array_exists_to_has]) return; auto * array_exists_function_node = node->as(); diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index f7dd542dbf4..9c99763cb56 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -12,6 +12,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool final; +} namespace { @@ -24,7 +28,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().final) + if (!getSettings()[Setting::final]) return; const auto * query_node = node->as(); diff --git a/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp b/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp index 6c4ce789993..39806fa6442 100644 --- a/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp +++ b/src/Analyzer/Passes/ConvertOrLikeChainPass.cpp @@ -26,6 +26,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_hyperscan; + extern const SettingsUInt64 max_hyperscan_regexp_length; + extern const SettingsUInt64 max_hyperscan_regexp_total_length; + extern const SettingsBool reject_expensive_hyperscan_regexps; + extern const SettingsBool optimize_or_like_chain; +} namespace { @@ -48,10 +56,8 @@ public: { const auto & settings = getSettings(); - return settings.optimize_or_like_chain - && settings.allow_hyperscan - && settings.max_hyperscan_regexp_length == 0 - && settings.max_hyperscan_regexp_total_length == 0; + return settings[Setting::optimize_or_like_chain] && settings[Setting::allow_hyperscan] && settings[Setting::max_hyperscan_regexp_length] == 0 + && settings[Setting::max_hyperscan_regexp_total_length] == 0; } void enterImpl(QueryTreeNodePtr & node) @@ -139,7 +145,11 @@ private: void ConvertOrLikeChainPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { const auto & settings = context->getSettingsRef(); - auto match_function_resolver = createInternalMultiMatchAnyOverloadResolver(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length, settings.reject_expensive_hyperscan_regexps); + auto match_function_resolver = createInternalMultiMatchAnyOverloadResolver( + settings[Setting::allow_hyperscan], + settings[Setting::max_hyperscan_regexp_length], + settings[Setting::max_hyperscan_regexp_total_length], + settings[Setting::reject_expensive_hyperscan_regexps]); auto or_function_resolver = createInternalFunctionOrOverloadResolver(); ConvertOrLikeChainVisitor visitor(std::move(or_function_resolver), std::move(match_function_resolver), std::move(context)); diff --git a/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp b/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp index f66d092429f..ddac55c1946 100644 --- a/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp +++ b/src/Analyzer/Passes/ConvertQueryToCNFPass.cpp @@ -19,6 +19,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool convert_query_to_cnf; + extern const SettingsBool optimize_append_index; + extern const SettingsBool optimize_substitute_columns; + extern const SettingsBool optimize_using_constraints; +} namespace { @@ -681,7 +688,7 @@ void optimizeWithConstraints(Analyzer::CNF & cnf, const QueryTreeNodes & table_e cnf.pushNotIntoFunctions(context); const auto & settings = context->getSettingsRef(); - if (settings.optimize_append_index) + if (settings[Setting::optimize_append_index]) addIndexConstraint(cnf, table_expressions, context); } @@ -693,7 +700,7 @@ void optimizeNode(QueryTreeNodePtr & node, const QueryTreeNodes & table_expressi if (!cnf) return; - if (settings.optimize_using_constraints) + if (settings[Setting::optimize_using_constraints]) optimizeWithConstraints(*cnf, table_expressions, context); auto new_node = cnf->toQueryTree(); @@ -731,7 +738,7 @@ public: optimize_filter(query_node->getPrewhere()); optimize_filter(query_node->getHaving()); - if (has_filter && settings.optimize_substitute_columns) + if (has_filter && settings[Setting::optimize_substitute_columns]) substituteColumns(*query_node, table_expressions, context); } }; @@ -741,7 +748,7 @@ public: void ConvertLogicalExpressionToCNFPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { const auto & settings = context->getSettingsRef(); - if (!settings.convert_query_to_cnf) + if (!settings[Setting::convert_query_to_cnf]) return; ConvertQueryToCNFVisitor visitor(std::move(context)); diff --git a/src/Analyzer/Passes/CountDistinctPass.cpp b/src/Analyzer/Passes/CountDistinctPass.cpp index 3f9185a07ca..6f92bdf6259 100644 --- a/src/Analyzer/Passes/CountDistinctPass.cpp +++ b/src/Analyzer/Passes/CountDistinctPass.cpp @@ -15,6 +15,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool count_distinct_optimization; +} namespace { @@ -27,7 +31,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().count_distinct_optimization) + if (!getSettings()[Setting::count_distinct_optimization]) return; auto * query_node = node->as(); diff --git a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp index eb615879893..3603745b279 100644 --- a/src/Analyzer/Passes/CrossToInnerJoinPass.cpp +++ b/src/Analyzer/Passes/CrossToInnerJoinPass.cpp @@ -21,6 +21,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 cross_to_inner_join_rewrite; +} namespace ErrorCodes { @@ -193,17 +197,14 @@ public: } private: - bool isEnabled() const - { - return getSettings().cross_to_inner_join_rewrite; - } + bool isEnabled() const { return getSettings()[Setting::cross_to_inner_join_rewrite]; } bool forceRewrite(JoinKind kind) const { if (kind == JoinKind::Cross) return false; /// Comma join can be forced to rewrite - return getSettings().cross_to_inner_join_rewrite >= 2; + return getSettings()[Setting::cross_to_inner_join_rewrite] >= 2; } QueryTreeNodePtr makeConjunction(const QueryTreeNodes & nodes) diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 6f1c3937880..f78eb2a0b76 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -25,6 +25,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool group_by_use_nulls; + extern const SettingsBool join_use_nulls; + extern const SettingsBool optimize_functions_to_subcolumns; +} namespace { @@ -257,7 +263,7 @@ public: void enterImpl(const QueryTreeNodePtr & node) { - if (!getSettings().optimize_functions_to_subcolumns) + if (!getSettings()[Setting::optimize_functions_to_subcolumns]) return; if (auto * table_node = node->as()) @@ -281,14 +287,14 @@ public: if (const auto * join_node = node->as()) { - can_wrap_result_columns_with_nullable |= getContext()->getSettingsRef().join_use_nulls; + can_wrap_result_columns_with_nullable |= getContext()->getSettingsRef()[Setting::join_use_nulls]; return; } if (const auto * query_node = node->as()) { if (query_node->isGroupByWithCube() || query_node->isGroupByWithRollup() || query_node->isGroupByWithGroupingSets()) - can_wrap_result_columns_with_nullable |= getContext()->getSettingsRef().group_by_use_nulls; + can_wrap_result_columns_with_nullable |= getContext()->getSettingsRef()[Setting::group_by_use_nulls]; return; } } @@ -419,7 +425,7 @@ public: void enterImpl(QueryTreeNodePtr & node) const { - if (!getSettings().optimize_functions_to_subcolumns) + if (!getSettings()[Setting::optimize_functions_to_subcolumns]) return; auto [function_node, first_argument_column_node, table_node] = getTypedNodesForOptimization(node, getContext()); diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp index f3b109a10ed..17a765a068b 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.cpp +++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp @@ -21,6 +21,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_syntax_fuse_functions; +} namespace ErrorCodes { @@ -44,7 +48,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_syntax_fuse_functions) + if (!getSettings()[Setting::optimize_syntax_fuse_functions]) return; auto * function_node = node->as(); diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index 91f06183523..dcf024bb46c 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool force_grouping_standard_compatibility; + extern const SettingsBool group_by_use_nulls; +} namespace ErrorCodes { @@ -72,41 +77,38 @@ public: FunctionOverloadResolverPtr grouping_function_resolver; bool add_grouping_set_column = false; - bool force_grouping_standard_compatibility = getSettings().force_grouping_standard_compatibility; + bool force_grouping_standard_compatibility = getSettings()[Setting::force_grouping_standard_compatibility]; size_t aggregation_keys_size = aggregation_key_to_index.size(); switch (group_by_kind) { case GroupByKind::ORDINARY: { - auto grouping_ordinary_function = std::make_shared(arguments_indexes, - force_grouping_standard_compatibility); + auto grouping_ordinary_function + = std::make_shared(arguments_indexes, force_grouping_standard_compatibility); grouping_function_resolver = std::make_shared(std::move(grouping_ordinary_function)); break; } case GroupByKind::ROLLUP: { - auto grouping_rollup_function = std::make_shared(arguments_indexes, - aggregation_keys_size, - force_grouping_standard_compatibility); + auto grouping_rollup_function = std::make_shared( + arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility); grouping_function_resolver = std::make_shared(std::move(grouping_rollup_function)); add_grouping_set_column = true; break; } case GroupByKind::CUBE: { - auto grouping_cube_function = std::make_shared(arguments_indexes, - aggregation_keys_size, - force_grouping_standard_compatibility); + auto grouping_cube_function = std::make_shared( + arguments_indexes, aggregation_keys_size, force_grouping_standard_compatibility); grouping_function_resolver = std::make_shared(std::move(grouping_cube_function)); add_grouping_set_column = true; break; } case GroupByKind::GROUPING_SETS: { - auto grouping_grouping_sets_function = std::make_shared(arguments_indexes, - grouping_sets_keys_indexes, - force_grouping_standard_compatibility); + auto grouping_grouping_sets_function = std::make_shared( + arguments_indexes, grouping_sets_keys_indexes, force_grouping_standard_compatibility); grouping_function_resolver = std::make_shared(std::move(grouping_grouping_sets_function)); add_grouping_set_column = true; break; @@ -147,7 +149,8 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) if (query_node_typed.hasGroupBy()) { /// It is expected by execution layer that if there are only 1 grouping set it will be removed - if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1 && !context->getSettingsRef().group_by_use_nulls) + if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1 + && !context->getSettingsRef()[Setting::group_by_use_nulls]) { auto grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front(); auto & grouping_set_list_node_typed = grouping_set_list_node->as(); diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp index cff98adb6af..92d70e30b09 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp @@ -10,6 +10,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_execute_multiif_columnar; + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool optimize_if_chain_to_multiif; + extern const SettingsBool use_variant_as_common_type; +} namespace { @@ -27,7 +34,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_if_chain_to_multiif) + if (!getSettings()[Setting::optimize_if_chain_to_multiif]) return; auto * function_node = node->as(); @@ -84,7 +91,8 @@ private: void IfChainToMultiIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { const auto & settings = context->getSettingsRef(); - auto multi_if_function_ptr = createInternalMultiIfOverloadResolver(settings.allow_execute_multiif_columnar, settings.allow_experimental_variant_type, settings.use_variant_as_common_type); + auto multi_if_function_ptr = createInternalMultiIfOverloadResolver( + settings[Setting::allow_execute_multiif_columnar], settings[Setting::allow_experimental_variant_type], settings[Setting::use_variant_as_common_type]); IfChainToMultiIfPassVisitor visitor(std::move(multi_if_function_ptr), std::move(context)); visitor.visit(query_tree_node); } diff --git a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp index f81327c5d55..b205efe4f88 100644 --- a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp +++ b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp @@ -16,6 +16,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_if_transform_strings_to_enum; +} namespace { @@ -101,7 +105,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_if_transform_strings_to_enum) + if (!getSettings()[Setting::optimize_if_transform_strings_to_enum]) return; auto * function_node = node->as(); diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index e136440556f..26a8b3f0d8f 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -16,6 +16,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 optimize_min_equality_disjunction_chain_length; + extern const SettingsUInt64 optimize_min_inequality_conjunction_chain_length; +} namespace ErrorCodes { @@ -531,7 +536,8 @@ private: for (auto & [expression, not_equals_functions] : node_to_not_equals_functions) { const auto & settings = getSettings(); - if (not_equals_functions.size() < settings.optimize_min_inequality_conjunction_chain_length && !expression.node->getResultType()->lowCardinality()) + if (not_equals_functions.size() < settings[Setting::optimize_min_inequality_conjunction_chain_length] + && !expression.node->getResultType()->lowCardinality()) { std::move(not_equals_functions.begin(), not_equals_functions.end(), std::back_inserter(and_operands)); continue; @@ -653,7 +659,8 @@ private: for (auto & [expression, equals_functions] : node_to_equals_functions) { const auto & settings = getSettings(); - if (equals_functions.size() < settings.optimize_min_equality_disjunction_chain_length && !expression.node->getResultType()->lowCardinality()) + if (equals_functions.size() < settings[Setting::optimize_min_equality_disjunction_chain_length] + && !expression.node->getResultType()->lowCardinality()) { std::move(equals_functions.begin(), equals_functions.end(), std::back_inserter(or_operands)); continue; diff --git a/src/Analyzer/Passes/MultiIfToIfPass.cpp b/src/Analyzer/Passes/MultiIfToIfPass.cpp index 0b441032b02..26a7270a2a2 100644 --- a/src/Analyzer/Passes/MultiIfToIfPass.cpp +++ b/src/Analyzer/Passes/MultiIfToIfPass.cpp @@ -8,6 +8,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool optimize_multiif_to_if; + extern const SettingsBool use_variant_as_common_type; +} namespace { @@ -25,7 +31,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_multiif_to_if) + if (!getSettings()[Setting::optimize_multiif_to_if]) return; auto * function_node = node->as(); @@ -57,7 +63,8 @@ private: void MultiIfToIfPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) { const auto & settings = context->getSettingsRef(); - auto if_function_ptr = createInternalFunctionIfOverloadResolver(settings.allow_experimental_variant_type, settings.use_variant_as_common_type); + auto if_function_ptr + = createInternalFunctionIfOverloadResolver(settings[Setting::allow_experimental_variant_type], settings[Setting::use_variant_as_common_type]); MultiIfToIfVisitor visitor(std::move(if_function_ptr), std::move(context)); visitor.visit(query_tree_node); } diff --git a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp index 02f1c93ea7f..1124dd35494 100644 --- a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp +++ b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp @@ -13,6 +13,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_normalize_count_variants; +} namespace { @@ -25,7 +29,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_normalize_count_variants) + if (!getSettings()[Setting::optimize_normalize_count_variants]) return; auto * function_node = node->as(); diff --git a/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp b/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp index 0f33c302265..f11d5fa003f 100644 --- a/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp +++ b/src/Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.cpp @@ -13,6 +13,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_time_filter_with_preimage; +} namespace ErrorCodes { @@ -58,7 +62,7 @@ public: {"greaterOrEquals", "lessOrEquals"}, }; - if (!getSettings().optimize_time_filter_with_preimage) + if (!getSettings()[Setting::optimize_time_filter_with_preimage]) return; const auto * function = node->as(); diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index bfb6aa1dd07..ceeecc65f10 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool group_by_use_nulls; + extern const SettingsBool optimize_group_by_function_keys; +} class OptimizeGroupByFunctionKeysVisitor : public InDepthQueryTreeVisitorWithContext { @@ -29,13 +34,13 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_group_by_function_keys) + if (!getSettings()[Setting::optimize_group_by_function_keys]) return; /// When group_by_use_nulls = 1 removing keys from GROUP BY can lead /// to unexpected types in some functions. /// See example in https://github.com/ClickHouse/ClickHouse/pull/61567#issuecomment-2018007887 - if (getSettings().group_by_use_nulls) + if (getSettings()[Setting::group_by_use_nulls]) return; auto * query = node->as(); diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index fc989003c02..9e1c21375fa 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -9,6 +9,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool group_by_use_nulls; + extern const SettingsBool optimize_injective_functions_in_group_by; +} namespace { @@ -23,14 +28,14 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_injective_functions_in_group_by) + if (!getSettings()[Setting::optimize_injective_functions_in_group_by]) return; /// Don't optimize injective functions when group_by_use_nulls=true, /// because in this case we make initial group by keys Nullable /// and eliminating some functions can cause issues with arguments Nullability /// during their execution. See examples in https://github.com/ClickHouse/ClickHouse/pull/61567#issuecomment-2008181143 - if (getSettings().group_by_use_nulls) + if (getSettings()[Setting::group_by_use_nulls]) return; auto * query = node->as(); diff --git a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp index 4f0271b4914..3b5ab1fef4d 100644 --- a/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp +++ b/src/Analyzer/Passes/OptimizeRedundantFunctionsInOrderByPass.cpp @@ -12,6 +12,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_redundant_functions_in_order_by; +} namespace { @@ -31,7 +35,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_redundant_functions_in_order_by) + if (!getSettings()[Setting::optimize_redundant_functions_in_order_by]) return; auto * query = node->as(); diff --git a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp index 091061ceb81..57f552f8f8b 100644 --- a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp +++ b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp @@ -20,6 +20,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_rewrite_aggregate_function_with_if; +} namespace { @@ -32,7 +36,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_rewrite_aggregate_function_with_if) + if (!getSettings()[Setting::optimize_rewrite_aggregate_function_with_if]) return; auto * function_node = node->as(); diff --git a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp index a42b4750f47..d4805fb5055 100644 --- a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp +++ b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp @@ -11,6 +11,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_arithmetic_operations_in_aggregate_functions; +} namespace { @@ -23,7 +27,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_arithmetic_operations_in_aggregate_functions) + if (!getSettings()[Setting::optimize_arithmetic_operations_in_aggregate_functions]) return; static const std::unordered_set func_supported = { diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index a987ced497a..363bbe2736d 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -14,6 +14,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool aggregate_functions_null_for_empty; + extern const SettingsBool optimize_rewrite_sum_if_to_count_if; +} namespace { @@ -26,7 +31,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_rewrite_sum_if_to_count_if) + if (!getSettings()[Setting::optimize_rewrite_sum_if_to_count_if]) return; auto * function_node = node->as(); @@ -56,7 +61,7 @@ public: return; const auto & constant_value_literal = constant_node->getValue(); - if (getSettings().aggregate_functions_null_for_empty) + if (getSettings()[Setting::aggregate_functions_null_for_empty]) return; /// Rewrite `sumIf(1, cond)` into `countIf(cond)` diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp index cb8ef35fc13..39d88491553 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp @@ -14,6 +14,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_injective_functions_inside_uniq; +} namespace { @@ -36,7 +40,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_injective_functions_inside_uniq) + if (!getSettings()[Setting::optimize_injective_functions_inside_uniq]) return; auto * function_node = node->as(); diff --git a/src/Analyzer/Passes/UniqToCountPass.cpp b/src/Analyzer/Passes/UniqToCountPass.cpp index 3f73322d1fa..bf09e8faf36 100644 --- a/src/Analyzer/Passes/UniqToCountPass.cpp +++ b/src/Analyzer/Passes/UniqToCountPass.cpp @@ -13,6 +13,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_uniq_to_count; +} namespace { @@ -123,7 +127,7 @@ public: void enterImpl(QueryTreeNodePtr & node) { - if (!getSettings().optimize_uniq_to_count) + if (!getSettings()[Setting::optimize_uniq_to_count]) return; auto * query_node = node->as(); diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 9754897d54d..e3cb193429c 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -50,6 +50,17 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool any_join_distinct_right_table_keys; + extern const SettingsJoinStrictness join_default_strictness; + extern const SettingsBool enable_order_by_all; + extern const SettingsUInt64 limit; + extern const SettingsUInt64 offset; + extern const SettingsBool use_variant_as_common_type; +} + namespace ErrorCodes { @@ -235,13 +246,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q UInt64 offset = 0; /// Remove global settings limit and offset - if (const auto & settings_ref = updated_context->getSettingsRef(); settings_ref.limit || settings_ref.offset) + if (const auto & settings_ref = updated_context->getSettingsRef(); settings_ref[Setting::limit] || settings_ref[Setting::offset]) { Settings settings = updated_context->getSettingsCopy(); - limit = settings.limit; - offset = settings.offset; - settings.limit = 0; - settings.offset = 0; + limit = settings[Setting::limit]; + offset = settings[Setting::offset]; + settings[Setting::limit] = 0; + settings[Setting::offset] = 0; updated_context->setSettings(settings); } @@ -268,7 +279,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q } } - const auto enable_order_by_all = updated_context->getSettingsRef().enable_order_by_all; + const auto enable_order_by_all = updated_context->getSettingsRef()[Setting::enable_order_by_all]; auto current_query_tree = std::make_shared(std::move(updated_context), std::move(settings_changes)); @@ -577,7 +588,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, co } else if (const auto * ast_literal = expression->as()) { - if (context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type) + if (context->getSettingsRef()[Setting::allow_experimental_variant_type] && context->getSettingsRef()[Setting::use_variant_as_common_type]) result = std::make_shared(ast_literal->value, applyVisitor(FieldToDataType(), ast_literal->value)); else result = std::make_shared(ast_literal->value); @@ -908,8 +919,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select join_expression = buildExpression(table_join.on_expression, context); const auto & settings = context->getSettingsRef(); - auto join_default_strictness = settings.join_default_strictness; - auto any_join_distinct_right_table_keys = settings.any_join_distinct_right_table_keys; + auto join_default_strictness = settings[Setting::join_default_strictness]; + auto any_join_distinct_right_table_keys = settings[Setting::any_join_distinct_right_table_keys]; JoinStrictness result_join_strictness = table_join.strictness; JoinKind result_join_kind = table_join.kind; diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.cpp b/src/Analyzer/Resolve/IdentifierResolveScope.cpp index eb3e2179440..84f6d500f1e 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.cpp +++ b/src/Analyzer/Resolve/IdentifierResolveScope.cpp @@ -7,6 +7,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool group_by_use_nulls; + extern const SettingsBool join_use_nulls; +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -32,12 +38,12 @@ IdentifierResolveScope::IdentifierResolveScope(QueryTreeNodePtr scope_node_, Ide else if (auto * query_node = scope_node->as()) { context = query_node->getContext(); - group_by_use_nulls = context->getSettingsRef().group_by_use_nulls && - (query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube()); + group_by_use_nulls = context->getSettingsRef()[Setting::group_by_use_nulls] + && (query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube()); } if (context) - join_use_nulls = context->getSettingsRef().join_use_nulls; + join_use_nulls = context->getSettingsRef()[Setting::join_use_nulls]; else if (parent_scope) join_use_nulls = parent_scope->join_use_nulls; diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index 80e7d1e4445..083b3a8f462 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -30,6 +30,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool single_join_prefer_left_table; +} + namespace ErrorCodes { extern const int UNKNOWN_IDENTIFIER; @@ -420,7 +426,7 @@ QueryTreeNodePtr IdentifierResolver::tryResolveTableIdentifierFromDatabaseCatalo if (!storage) return {}; - auto storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + auto storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); auto storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), context); auto result = std::make_shared(std::move(storage), std::move(storage_lock), std::move(storage_snapshot)); if (is_temporary_table) @@ -1155,7 +1161,7 @@ QueryTreeNodePtr IdentifierResolver::tryResolveIdentifierFromJoin(const Identifi resolved_identifier = left_resolved_identifier; } } - else if (scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) + else if (scope.joins_count == 1 && scope.context->getSettingsRef()[Setting::single_join_prefer_left_table]) { resolved_side = JoinTableSide::Left; resolved_identifier = left_resolved_identifier; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index a526805fd4d..3f88106e8fa 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -77,6 +77,35 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool aggregate_functions_null_for_empty; + extern const SettingsBool analyzer_compatibility_join_using_top_level_identifier; + extern const SettingsBool asterisk_include_alias_columns; + extern const SettingsBool asterisk_include_materialized_columns; + extern const SettingsString count_distinct_implementation; + extern const SettingsBool enable_global_with_statement; + extern const SettingsBool enable_order_by_all; + extern const SettingsBool enable_positional_arguments; + extern const SettingsBool enable_scalar_subquery_optimization; + extern const SettingsBool extremes; + extern const SettingsBool force_grouping_standard_compatibility; + extern const SettingsBool format_display_secrets_in_show_and_select; + extern const SettingsBool joined_subquery_requires_alias; + extern const SettingsUInt64 max_bytes_in_set; + extern const SettingsUInt64 max_expanded_ast_elements; + extern const SettingsUInt64 max_result_rows; + extern const SettingsUInt64 max_rows_in_set; + extern const SettingsUInt64 max_subquery_depth; + extern const SettingsBool prefer_column_name_to_alias; + extern const SettingsBool rewrite_count_distinct_if_with_count_distinct_implementation; + extern const SettingsOverflowMode set_overflow_mode; + extern const SettingsBool single_join_prefer_left_table; + extern const SettingsBool transform_null_in; + extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; +} + + namespace ErrorCodes { extern const int UNSUPPORTED_METHOD; @@ -506,8 +535,8 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden auto subquery_context = Context::createCopy(context); Settings subquery_settings = context->getSettingsCopy(); - subquery_settings.max_result_rows = 1; - subquery_settings.extremes = false; + subquery_settings[Setting::max_result_rows] = 1; + subquery_settings[Setting::extremes] = false; subquery_context->setSettings(subquery_settings); /// When execute `INSERT INTO t WITH ... SELECT ...`, it may lead to `Unknown columns` /// exception with this settings enabled(https://github.com/ClickHouse/ClickHouse/issues/52494). @@ -627,10 +656,8 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden auto * nearest_query_scope = scope.getNearestQueryScope(); /// Always convert to literals when there is no query context - if (!context->getSettingsRef().enable_scalar_subquery_optimization || - !useless_literal_types.contains(scalar_type_name) || - !context->hasQueryContext() || - !nearest_query_scope) + if (!context->getSettingsRef()[Setting::enable_scalar_subquery_optimization] || !useless_literal_types.contains(scalar_type_name) + || !context->hasQueryContext() || !nearest_query_scope) { auto constant_value = std::make_shared(std::move(scalar_value), scalar_type); auto constant_node = std::make_shared(constant_value, node); @@ -726,7 +753,7 @@ void QueryAnalyzer::mergeWindowWithParentWindow(const QueryTreeNodePtr & window_ void QueryAnalyzer::replaceNodesWithPositionalArguments(QueryTreeNodePtr & node_list, const QueryTreeNodes & projection_nodes, IdentifierResolveScope & scope) { const auto & settings = scope.context->getSettingsRef(); - if (!settings.enable_positional_arguments || scope.context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY) + if (!settings[Setting::enable_positional_arguments] || scope.context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY) return; auto & node_list_typed = node_list->as(); @@ -843,7 +870,7 @@ void QueryAnalyzer::validateTableExpressionModifiers(const QueryTreeNodePtr & ta void QueryAnalyzer::validateJoinTableExpressionWithoutAlias(const QueryTreeNodePtr & join_node, const QueryTreeNodePtr & table_expression_node, IdentifierResolveScope & scope) { - if (!scope.context->getSettingsRef().joined_subquery_requires_alias) + if (!scope.context->getSettingsRef()[Setting::joined_subquery_requires_alias]) return; bool table_expression_has_alias = table_expression_node->hasAlias(); @@ -938,7 +965,7 @@ void QueryAnalyzer::expandGroupByAll(QueryNode & query_tree_node_typed) void QueryAnalyzer::expandOrderByAll(QueryNode & query_tree_node_typed, const Settings & settings) { - if (!settings.enable_order_by_all || !query_tree_node_typed.isOrderByAll()) + if (!settings[Setting::enable_order_by_all] || !query_tree_node_typed.isOrderByAll()) return; auto * all_node = query_tree_node_typed.getOrderBy().getNodes()[0]->as(); @@ -989,12 +1016,14 @@ std::string QueryAnalyzer::rewriteAggregateFunctionNameIfNeeded( if (aggregate_function_name_lowercase == "countdistinct") { - result_aggregate_function_name = settings.count_distinct_implementation; + result_aggregate_function_name = settings[Setting::count_distinct_implementation]; } - else if (aggregate_function_name_lowercase == "countifdistinct" || - (settings.rewrite_count_distinct_if_with_count_distinct_implementation && aggregate_function_name_lowercase == "countdistinctif")) + else if ( + aggregate_function_name_lowercase == "countifdistinct" + || (settings[Setting::rewrite_count_distinct_if_with_count_distinct_implementation] + && aggregate_function_name_lowercase == "countdistinctif")) { - result_aggregate_function_name = settings.count_distinct_implementation; + result_aggregate_function_name = settings[Setting::count_distinct_implementation]; result_aggregate_function_name += "If"; } else if (aggregate_function_name_lowercase.ends_with("ifdistinct")) @@ -1004,7 +1033,7 @@ std::string QueryAnalyzer::rewriteAggregateFunctionNameIfNeeded( result_aggregate_function_name = result_aggregate_function_name.substr(0, prefix_length) + "DistinctIf"; } - bool need_add_or_null = settings.aggregate_functions_null_for_empty && !result_aggregate_function_name.ends_with("OrNull"); + bool need_add_or_null = settings[Setting::aggregate_functions_null_for_empty] && !result_aggregate_function_name.ends_with("OrNull"); if (need_add_or_null) { auto properties = AggregateFunctionFactory::instance().tryGetProperties(result_aggregate_function_name, action); @@ -1215,7 +1244,7 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierInParentScopes(const } } - if (!scope.context->getSettingsRef().enable_global_with_statement) + if (!scope.context->getSettingsRef()[Setting::enable_global_with_statement]) return {}; /** Nested subqueries cannot access outer subqueries table expressions from JOIN tree because @@ -1347,7 +1376,7 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifier(const IdentifierLook if (!resolve_result.resolved_identifier) { - bool prefer_column_name_to_alias = scope.context->getSettingsRef().prefer_column_name_to_alias; + bool prefer_column_name_to_alias = scope.context->getSettingsRef()[Setting::prefer_column_name_to_alias]; if (identifier_lookup.isExpressionLookup()) { @@ -1558,10 +1587,10 @@ GetColumnsOptions QueryAnalyzer::buildGetColumnsOptions(QueryTreeNodePtr & match const auto & settings = context->getSettingsRef(); - if (settings.asterisk_include_alias_columns) + if (settings[Setting::asterisk_include_alias_columns]) get_columns_options_kind |= GetColumnsOptions::Kind::Aliases; - if (settings.asterisk_include_materialized_columns) + if (settings[Setting::asterisk_include_materialized_columns]) get_columns_options_kind |= GetColumnsOptions::Kind::Materialized; } @@ -2810,7 +2839,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi allow_table_expressions /*allow_table_expression*/); /// Mask arguments if needed - if (!scope.context->getSettingsRef().format_display_secrets_in_show_and_select) + if (!scope.context->getSettingsRef()[Setting::format_display_secrets_in_show_and_select]) { if (FunctionSecretArgumentsFinder::Result secret_arguments = FunctionSecretArgumentsFinderTreeNode(*function_node_ptr).getResult(); secret_arguments.count) { @@ -2834,7 +2863,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi if (is_special_function_in) { checkFunctionNodeHasEmptyNullsAction(function_node); - if (scope.context->getSettingsRef().transform_null_in) + if (scope.context->getSettingsRef()[Setting::transform_null_in]) { static constexpr std::array, 4> in_function_to_replace_null_in_function_map = {{ @@ -3134,7 +3163,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi function_arguments_size); checkFunctionNodeHasEmptyNullsAction(function_node); - bool force_grouping_standard_compatibility = scope.context->getSettingsRef().force_grouping_standard_compatibility; + bool force_grouping_standard_compatibility = scope.context->getSettingsRef()[Setting::force_grouping_standard_compatibility]; auto grouping_function = std::make_shared(force_grouping_standard_compatibility); auto grouping_function_adaptor = std::make_shared(std::move(grouping_function)); function_node.resolveAsFunction(grouping_function_adaptor->build(argument_columns)); @@ -3388,14 +3417,12 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi const auto & settings = scope.context->getSettingsRef(); - auto result_block = getSetElementsForConstantValue(first_argument_constant_type, - second_argument_constant_literal, - second_argument_constant_type, - settings.transform_null_in); + auto result_block = getSetElementsForConstantValue( + first_argument_constant_type, second_argument_constant_literal, second_argument_constant_type, settings[Setting::transform_null_in]); - SizeLimits size_limits_for_set = {settings.max_rows_in_set, settings.max_bytes_in_set, settings.set_overflow_mode}; + SizeLimits size_limits_for_set = {settings[Setting::max_rows_in_set], settings[Setting::max_bytes_in_set], settings[Setting::set_overflow_mode]}; - auto set = std::make_shared(size_limits_for_set, 0, settings.transform_null_in); + auto set = std::make_shared(size_limits_for_set, 0, settings[Setting::transform_null_in]); set->setHeader(result_block.cloneEmpty().getColumnsWithTypeAndName()); set->insertFromBlock(result_block.getColumnsWithTypeAndName()); @@ -3826,10 +3853,10 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( } } - validateTreeSize(node, scope.context->getSettingsRef().max_expanded_ast_elements, node_to_tree_size); + validateTreeSize(node, scope.context->getSettingsRef()[Setting::max_expanded_ast_elements], node_to_tree_size); /// Lambda can be inside the aggregate function, so we should check parent scopes. - /// Most likely only the root scope can have an arrgegate function, but let's check all just in case. + /// Most likely only the root scope can have an aggregate function, but let's check all just in case. bool in_aggregate_function_scope = false; for (const auto * scope_ptr = &scope; scope_ptr; scope_ptr = scope_ptr->parent_scope) in_aggregate_function_scope = in_aggregate_function_scope || scope_ptr->expressions_in_resolve_process_stack.hasAggregateFunction(); @@ -4473,9 +4500,9 @@ void QueryAnalyzer::initializeTableExpressionData(const QueryTreeNodePtr & table if (auto * scope_query_node = scope.scope_node->as()) { auto left_table_expression = extractLeftTableExpression(scope_query_node->getJoinTree()); - if (table_expression_node.get() == left_table_expression.get() && - scope.joins_count == 1 && scope.context->getSettingsRef().single_join_prefer_left_table) - table_expression_data.should_qualify_columns = false; + if (table_expression_node.get() == left_table_expression.get() && scope.joins_count == 1 + && scope.context->getSettingsRef()[Setting::single_join_prefer_left_table]) + table_expression_data.should_qualify_columns = false; } scope.table_expression_node_to_data.emplace(table_expression_node, std::move(table_expression_data)); @@ -4672,11 +4699,10 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, table_function_ptr->parseArguments(table_function_ast, scope_context); - uint64_t use_structure_from_insertion_table_in_table_functions = scope_context->getSettingsRef().use_structure_from_insertion_table_in_table_functions; - if (!nested_table_function && - use_structure_from_insertion_table_in_table_functions && - scope_context->hasInsertionTable() && - table_function_ptr->needStructureHint()) + uint64_t use_structure_from_insertion_table_in_table_functions + = scope_context->getSettingsRef()[Setting::use_structure_from_insertion_table_in_table_functions]; + if (!nested_table_function && use_structure_from_insertion_table_in_table_functions && scope_context->hasInsertionTable() + && table_function_ptr->needStructureHint()) { const auto & insertion_table = scope_context->getInsertionTable(); if (!insertion_table.empty()) @@ -4806,8 +4832,8 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, if (!structure_hint.empty()) table_function_ptr->setStructureHint(structure_hint); - - } else if (use_structure_from_insertion_table_in_table_functions == 1) + } + else if (use_structure_from_insertion_table_in_table_functions == 1) throw Exception(ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns in insert table less than required by SELECT expression."); } } @@ -4931,7 +4957,7 @@ void QueryAnalyzer::resolveArrayJoin(QueryTreeNodePtr & array_join_node, Identif void QueryAnalyzer::checkDuplicateTableNamesOrAlias(const QueryTreeNodePtr & join_node, QueryTreeNodePtr & left_table_expr, QueryTreeNodePtr & right_table_expr, IdentifierResolveScope & scope) { Names column_names; - if (!scope.context->getSettingsRef().joined_subquery_requires_alias) + if (!scope.context->getSettingsRef()[Setting::joined_subquery_requires_alias]) return; if (join_node->as().getKind() != JoinKind::Paste) @@ -5051,7 +5077,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS * despite the fact that column from USING could be resolved from left table. * It's compatibility with a default behavior for old analyzer. */ - if (settings.analyzer_compatibility_join_using_top_level_identifier) + if (settings[Setting::analyzer_compatibility_join_using_top_level_identifier]) result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope); IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; @@ -5070,7 +5096,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS { String extra_message; const QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; - if (settings.analyzer_compatibility_join_using_top_level_identifier && query_node) + if (settings[Setting::analyzer_compatibility_join_using_top_level_identifier] && query_node) { for (const auto & projection_node : query_node->getProjection().getNodes()) { @@ -5250,11 +5276,9 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, */ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, IdentifierResolveScope & scope) { - size_t max_subquery_depth = scope.context->getSettingsRef().max_subquery_depth; + size_t max_subquery_depth = scope.context->getSettingsRef()[Setting::max_subquery_depth]; if (max_subquery_depth && scope.subquery_depth > max_subquery_depth) - throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, - "Too deep subqueries. Maximum: {}", - max_subquery_depth); + throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, "Too deep subqueries. Maximum: {}", max_subquery_depth); auto & query_node_typed = query_node->as(); @@ -5588,7 +5612,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier expandGroupByAll(query_node_typed); validateFilters(query_node); - validateAggregates(query_node, { .group_by_use_nulls = scope.group_by_use_nulls }); + validateAggregates(query_node, {.group_by_use_nulls = scope.group_by_use_nulls}); for (const auto & column : projection_columns) { diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index d5a4b06f652..506bb1c08ef 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -14,6 +14,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} TableNode::TableNode(StoragePtr storage_, StorageID storage_id_, TableLockHolder storage_lock_, StorageSnapshotPtr storage_snapshot_) : IQueryTreeNode(children_size) @@ -29,9 +33,10 @@ TableNode::TableNode(StoragePtr storage_, TableLockHolder storage_lock_, Storage } TableNode::TableNode(StoragePtr storage_, const ContextPtr & context) - : TableNode(storage_, - storage_->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout), - storage_->getStorageSnapshot(storage_->getInMemoryMetadataPtr(), context)) + : TableNode( + storage_, + storage_->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]), + storage_->getStorageSnapshot(storage_->getInMemoryMetadataPtr(), context)) { } @@ -39,7 +44,7 @@ void TableNode::updateStorage(StoragePtr storage_value, const ContextPtr & conte { storage = std::move(storage_value); storage_id = storage->getStorageID(); - storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + storage_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); storage_snapshot = storage->getStorageSnapshot(storage->getInMemoryMetadataPtr(), context); } diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index e5f372b7368..c737822282c 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -36,6 +36,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool extremes; + extern const SettingsUInt64 max_result_bytes; + extern const SettingsUInt64 max_result_rows; +} namespace ErrorCodes { @@ -868,10 +874,10 @@ void updateContextForSubqueryExecution(ContextMutablePtr & mutable_context) * which are checked separately (in the Set, Join objects). */ Settings subquery_settings = mutable_context->getSettingsCopy(); - subquery_settings.max_result_rows = 0; - subquery_settings.max_result_bytes = 0; + subquery_settings[Setting::max_result_rows] = 0; + subquery_settings[Setting::max_result_bytes] = 0; /// The calculation of extremes does not make sense and is not necessary (if you do it, then the extremes of the subquery can be taken for whole query). - subquery_settings.extremes = false; + subquery_settings[Setting::extremes] = false; mutable_context->setSettings(subquery_settings); } diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 3ee7a05600e..2024d7772f9 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -35,6 +35,13 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 backup_restore_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_retry_max_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_max_retries; + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -105,9 +112,9 @@ BackupEntriesCollector::BackupEntriesCollector( , compare_collected_metadata(context->getConfigRef().getBool("backups.compare_collected_metadata", true)) , log(getLogger("BackupEntriesCollector")) , global_zookeeper_retries_info( - context->getSettingsRef().backup_restore_keeper_max_retries, - context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, - context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms) + context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], + context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], + context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms]) , threadpool(threadpool_) { } @@ -653,7 +660,7 @@ void BackupEntriesCollector::lockTablesForReading() checkIsQueryCancelled(); - table_info.table_lock = storage->tryLockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + table_info.table_lock = storage->tryLockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); } std::erase_if( diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 687096d0404..506133f7171 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -25,6 +25,16 @@ namespace fs = std::filesystem; namespace DB { + +namespace Setting +{ + extern const SettingsUInt64 backup_restore_s3_retry_attempts; + extern const SettingsBool enable_s3_requests_logging; + extern const SettingsBool s3_disable_checksum; + extern const SettingsUInt64 s3_max_connections; + extern const SettingsUInt64 s3_max_redirects; +} + namespace ErrorCodes { extern const int S3_ERROR; @@ -55,16 +65,16 @@ namespace S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( settings.auth_settings.region, context->getRemoteHostFilter(), - static_cast(local_settings.s3_max_redirects), - static_cast(local_settings.backup_restore_s3_retry_attempts), - local_settings.enable_s3_requests_logging, + static_cast(local_settings[Setting::s3_max_redirects]), + static_cast(local_settings[Setting::backup_restore_s3_retry_attempts]), + local_settings[Setting::enable_s3_requests_logging], /* for_disk_s3 = */ false, request_settings.get_request_throttler, request_settings.put_request_throttler, s3_uri.uri.getScheme()); client_configuration.endpointOverride = s3_uri.endpoint; - client_configuration.maxConnections = static_cast(global_settings.s3_max_connections); + client_configuration.maxConnections = static_cast(global_settings[Setting::s3_max_connections]); /// Increase connect timeout client_configuration.connectTimeoutMs = 10 * 1000; /// Requests in backups can be extremely long, set to one hour @@ -74,7 +84,7 @@ namespace S3::ClientSettings client_settings{ .use_virtual_addressing = s3_uri.is_virtual_hosted_style, - .disable_checksum = local_settings.s3_disable_checksum, + .disable_checksum = local_settings[Setting::s3_disable_checksum], .gcs_issue_compose_request = context->getConfigRef().getBool("s3.gcs_issue_compose_request", false), .is_s3express_bucket = S3::isS3ExpressEndpoint(s3_uri.endpoint), }; diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 0b93ae6d547..79f5d9eeb47 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -43,6 +43,15 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 backup_restore_batch_size_for_keeper_multiread; + extern const SettingsUInt64 backup_restore_keeper_max_retries; + extern const SettingsUInt64 backup_restore_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_retry_max_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_fault_injection_seed; + extern const SettingsFloat backup_restore_keeper_fault_injection_probability; +} namespace ErrorCodes { @@ -98,12 +107,12 @@ namespace RestoreCoordinationRemote::RestoreKeeperSettings keeper_settings { - .keeper_max_retries = context->getSettingsRef().backup_restore_keeper_max_retries, - .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, - .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms, - .batch_size_for_keeper_multiread = context->getSettingsRef().backup_restore_batch_size_for_keeper_multiread, - .keeper_fault_injection_probability = context->getSettingsRef().backup_restore_keeper_fault_injection_probability, - .keeper_fault_injection_seed = context->getSettingsRef().backup_restore_keeper_fault_injection_seed + .keeper_max_retries = context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], + .keeper_retry_initial_backoff_ms = context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], + .keeper_retry_max_backoff_ms = context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms], + .batch_size_for_keeper_multiread = context->getSettingsRef()[Setting::backup_restore_batch_size_for_keeper_multiread], + .keeper_fault_injection_probability = context->getSettingsRef()[Setting::backup_restore_keeper_fault_injection_probability], + .keeper_fault_injection_seed = context->getSettingsRef()[Setting::backup_restore_keeper_fault_injection_seed] }; auto all_hosts = BackupSettings::Util::filterHostIDs( diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 5e6beec791d..021d4208fb9 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -36,6 +36,11 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} + namespace ErrorCodes { extern const int BACKUP_ENTRY_NOT_FOUND; @@ -949,7 +954,7 @@ void RestorerFromBackup::checkTable(const QualifiedTableName & table_name) StoragePtr storage = database->getTable(resolved_id.table_name, context); table_info.storage = storage; - table_info.table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + table_info.table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); if (!restore_settings.allow_different_table_def && !table_info.is_predefined_table) { diff --git a/src/Backups/WithRetries.cpp b/src/Backups/WithRetries.cpp index 9f22085f5a9..772f746e40a 100644 --- a/src/Backups/WithRetries.cpp +++ b/src/Backups/WithRetries.cpp @@ -5,20 +5,30 @@ namespace DB { - +namespace Setting +{ + extern const SettingsUInt64 backup_restore_keeper_max_retries; + extern const SettingsUInt64 backup_restore_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 backup_restore_keeper_retry_max_backoff_ms; + extern const SettingsUInt64 backup_restore_batch_size_for_keeper_multiread; + extern const SettingsFloat backup_restore_keeper_fault_injection_probability; + extern const SettingsUInt64 backup_restore_keeper_fault_injection_seed; + extern const SettingsUInt64 backup_restore_keeper_value_max_size; + extern const SettingsUInt64 backup_restore_batch_size_for_keeper_multi; +} WithRetries::KeeperSettings WithRetries::KeeperSettings::fromContext(ContextPtr context) { return { - .keeper_max_retries = context->getSettingsRef().backup_restore_keeper_max_retries, - .keeper_retry_initial_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_initial_backoff_ms, - .keeper_retry_max_backoff_ms = context->getSettingsRef().backup_restore_keeper_retry_max_backoff_ms, - .batch_size_for_keeper_multiread = context->getSettingsRef().backup_restore_batch_size_for_keeper_multiread, - .keeper_fault_injection_probability = context->getSettingsRef().backup_restore_keeper_fault_injection_probability, - .keeper_fault_injection_seed = context->getSettingsRef().backup_restore_keeper_fault_injection_seed, - .keeper_value_max_size = context->getSettingsRef().backup_restore_keeper_value_max_size, - .batch_size_for_keeper_multi = context->getSettingsRef().backup_restore_batch_size_for_keeper_multi, + .keeper_max_retries = context->getSettingsRef()[Setting::backup_restore_keeper_max_retries], + .keeper_retry_initial_backoff_ms = context->getSettingsRef()[Setting::backup_restore_keeper_retry_initial_backoff_ms], + .keeper_retry_max_backoff_ms = context->getSettingsRef()[Setting::backup_restore_keeper_retry_max_backoff_ms], + .batch_size_for_keeper_multiread = context->getSettingsRef()[Setting::backup_restore_batch_size_for_keeper_multiread], + .keeper_fault_injection_probability = context->getSettingsRef()[Setting::backup_restore_keeper_fault_injection_probability], + .keeper_fault_injection_seed = context->getSettingsRef()[Setting::backup_restore_keeper_fault_injection_seed], + .keeper_value_max_size = context->getSettingsRef()[Setting::backup_restore_keeper_value_max_size], + .batch_size_for_keeper_multi = context->getSettingsRef()[Setting::backup_restore_batch_size_for_keeper_multi], }; } diff --git a/src/BridgeHelper/CatBoostLibraryBridgeHelper.h b/src/BridgeHelper/CatBoostLibraryBridgeHelper.h index 5d5c6d01705..f9694359c05 100644 --- a/src/BridgeHelper/CatBoostLibraryBridgeHelper.h +++ b/src/BridgeHelper/CatBoostLibraryBridgeHelper.h @@ -10,6 +10,8 @@ namespace DB { +struct ColumnWithTypeAndName; +using ColumnsWithTypeAndName = std::vector; class CatBoostLibraryBridgeHelper final : public LibraryBridgeHelper { diff --git a/src/BridgeHelper/IBridgeHelper.h b/src/BridgeHelper/IBridgeHelper.h index 8ce1c0e143a..a9662815a1e 100644 --- a/src/BridgeHelper/IBridgeHelper.h +++ b/src/BridgeHelper/IBridgeHelper.h @@ -1,11 +1,12 @@ #pragma once #include -#include -#include -#include #include +#include +#include +#include +#include namespace DB { diff --git a/src/BridgeHelper/LibraryBridgeHelper.cpp b/src/BridgeHelper/LibraryBridgeHelper.cpp index 12ca5564e80..0f8d50309cd 100644 --- a/src/BridgeHelper/LibraryBridgeHelper.cpp +++ b/src/BridgeHelper/LibraryBridgeHelper.cpp @@ -6,12 +6,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds http_receive_timeout; +} LibraryBridgeHelper::LibraryBridgeHelper(ContextPtr context_) : IBridgeHelper(context_) , config(context_->getConfigRef()) , log(getLogger("LibraryBridgeHelper")) - , http_timeout(context_->getGlobalContext()->getSettingsRef().http_receive_timeout.value) + , http_timeout(context_->getGlobalContext()->getSettingsRef()[Setting::http_receive_timeout].value) , bridge_host(config.getString("library_bridge.host", DEFAULT_HOST)) , bridge_port(config.getUInt("library_bridge.port", DEFAULT_PORT)) , http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_->getSettingsRef(), context_->getServerSettings().keep_alive_timeout)) diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp index 0649aa5f5d7..df78b890c5e 100644 --- a/src/Client/ClientApplicationBase.cpp +++ b/src/Client/ClientApplicationBase.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 5c86d911f96..a61ea4c8156 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -93,6 +92,21 @@ using namespace std::literals; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsBool async_insert; + extern const SettingsDialect dialect; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsUInt64 output_format_pretty_max_rows; + extern const SettingsUInt64 output_format_pretty_max_value_width; + extern const SettingsBool partial_result_on_first_cancel; + extern const SettingsBool throw_if_no_data_to_insert; +} namespace ErrorCodes { @@ -295,24 +309,24 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting size_t max_length = 0; if (!allow_multi_statements) - max_length = settings.max_query_size; + max_length = settings[Setting::max_query_size]; - const Dialect & dialect = settings.dialect; + const Dialect dialect = settings[Setting::dialect]; if (dialect == Dialect::kusto) - parser = std::make_unique(end, settings.allow_settings_after_format_in_insert); + parser = std::make_unique(end, settings[Setting::allow_settings_after_format_in_insert]); else if (dialect == Dialect::prql) - parser = std::make_unique(max_length, settings.max_parser_depth, settings.max_parser_backtracks); + parser = std::make_unique(max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); else - parser = std::make_unique(end, settings.allow_settings_after_format_in_insert); + parser = std::make_unique(end, settings[Setting::allow_settings_after_format_in_insert]); if (is_interactive || ignore_error) { String message; if (dialect == Dialect::kusto) - res = tryParseKQLQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks, true); + res = tryParseKQLQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks], true); else - res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks, true); + res = tryParseQuery(*parser, pos, end, message, true, "", allow_multi_statements, max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks], true); if (!res) { @@ -323,9 +337,9 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting else { if (dialect == Dialect::kusto) - res = parseKQLQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks); + res = parseKQLQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); else - res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth, settings.max_parser_backtracks); + res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } if (is_interactive) @@ -346,7 +360,8 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting /// Consumes trailing semicolons and tries to consume the same-line trailing comment. -void ClientBase::adjustQueryEnd(const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth, uint32_t max_parser_backtracks) +void ClientBase::adjustQueryEnd( + const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth, uint32_t max_parser_backtracks) { // We have to skip the trailing semicolon that might be left // after VALUES parsing or just after a normal semicolon-terminated query. @@ -672,16 +687,16 @@ void ClientBase::adjustSettings() /// Do not limit pretty format output in case of --pager specified or in case of stdout is not a tty. if (!pager.empty() || !stdout_is_a_tty) { - if (!global_context->getSettingsRef().output_format_pretty_max_rows.changed) + if (!global_context->getSettingsRef()[Setting::output_format_pretty_max_rows].changed) { - settings.output_format_pretty_max_rows = std::numeric_limits::max(); - settings.output_format_pretty_max_rows.changed = false; + settings[Setting::output_format_pretty_max_rows] = std::numeric_limits::max(); + settings[Setting::output_format_pretty_max_rows].changed = false; } - if (!global_context->getSettingsRef().output_format_pretty_max_value_width.changed) + if (!global_context->getSettingsRef()[Setting::output_format_pretty_max_value_width].changed) { - settings.output_format_pretty_max_value_width = std::numeric_limits::max(); - settings.output_format_pretty_max_value_width.changed = false; + settings[Setting::output_format_pretty_max_value_width] = std::numeric_limits::max(); + settings[Setting::output_format_pretty_max_value_width].changed = false; } } @@ -767,18 +782,17 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration() default_input_format = "TSV"; } - format_max_block_size = getClientConfiguration().getUInt64("format_max_block_size", - global_context->getSettingsRef().max_block_size); + format_max_block_size = getClientConfiguration().getUInt64("format_max_block_size", global_context->getSettingsRef()[Setting::max_block_size]); /// Setting value from cmd arg overrides one from config - if (global_context->getSettingsRef().max_insert_block_size.changed) + if (global_context->getSettingsRef()[Setting::max_insert_block_size].changed) { - insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size; + insert_format_max_block_size = global_context->getSettingsRef()[Setting::max_insert_block_size]; } else { - insert_format_max_block_size = getClientConfiguration().getUInt64("insert_format_max_block_size", - global_context->getSettingsRef().max_insert_block_size); + insert_format_max_block_size + = getClientConfiguration().getUInt64("insert_format_max_block_size", global_context->getSettingsRef()[Setting::max_insert_block_size]); } } @@ -878,7 +892,7 @@ bool ClientBase::isSyncInsertWithData(const ASTInsertQuery & insert_query, const if (insert_query.settings_ast) settings.applyChanges(insert_query.settings_ast->as()->changes); - return !settings.async_insert; + return !settings[Setting::async_insert]; } void ClientBase::processTextAsSingleQuery(const String & full_query) @@ -1031,7 +1045,7 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa } const auto & settings = client_context->getSettingsRef(); - const Int32 signals_before_stop = settings.partial_result_on_first_cancel ? 2 : 1; + const Int32 signals_before_stop = settings[Setting::partial_result_on_first_cancel] ? 2 : 1; int retries_left = 10; while (retries_left) @@ -1059,11 +1073,11 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa catch (const NetException &) { // We still want to attempt to process whatever we already received or can receive (socket receive buffer can be not empty) - receiveResult(parsed_query, signals_before_stop, settings.partial_result_on_first_cancel); + receiveResult(parsed_query, signals_before_stop, settings[Setting::partial_result_on_first_cancel]); throw; } - receiveResult(parsed_query, signals_before_stop, settings.partial_result_on_first_cancel); + receiveResult(parsed_query, signals_before_stop, settings[Setting::partial_result_on_first_cancel]); break; } @@ -1491,7 +1505,7 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars if ((!parsed_insert_query.data && !parsed_insert_query.infile) && (is_interactive || (!stdin_is_a_tty && !isStdinNotEmptyAndValid(std_in)))) { const auto & settings = client_context->getSettingsRef(); - if (settings.throw_if_no_data_to_insert) + if (settings[Setting::throw_if_no_data_to_insert]) throw Exception(ErrorCodes::NO_DATA_TO_INSERT, "No data to insert"); else return; @@ -1609,14 +1623,14 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des auto metadata = storage->getInMemoryMetadataPtr(); QueryPlan plan; storage->read( - plan, - sample.getNames(), - storage->getStorageSnapshot(metadata, client_context), - query_info, - client_context, - {}, - client_context->getSettingsRef().max_block_size, - getNumberOfPhysicalCPUCores()); + plan, + sample.getNames(), + storage->getStorageSnapshot(metadata, client_context), + query_info, + client_context, + {}, + client_context->getSettingsRef()[Setting::max_block_size], + getNumberOfPhysicalCPUCores()); auto builder = plan.buildQueryPipeline( QueryPlanOptimizationSettings::fromContext(client_context), @@ -1953,7 +1967,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (insert && insert->select) insert->tryFindInputFunction(input_function); - bool is_async_insert_with_inlined_data = client_context->getSettingsRef().async_insert && insert && insert->hasInlinedData(); + bool is_async_insert_with_inlined_data = client_context->getSettingsRef()[Setting::async_insert] && insert && insert->hasInlinedData(); if (is_async_insert_with_inlined_data) { @@ -2038,11 +2052,11 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin error_stream << progress_indication.elapsedSeconds() << "\n"; const auto & print_memory_mode = config.getString("print-memory-to-stderr", ""); - auto peak_memeory_usage = std::max(progress_indication.getMemoryUsage().peak, 0); + auto peak_memory_usage = std::max(progress_indication.getMemoryUsage().peak, 0); if (print_memory_mode == "default") - error_stream << peak_memeory_usage << "\n"; + error_stream << peak_memory_usage << "\n"; else if (print_memory_mode == "readable") - error_stream << formatReadableSizeWithBinarySuffix(peak_memeory_usage) << "\n"; + error_stream << formatReadableSizeWithBinarySuffix(peak_memory_usage) << "\n"; } if (!is_interactive && getClientConfiguration().getBool("print-num-processed-rows", false)) @@ -2075,8 +2089,8 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( if (this_query_begin >= all_queries_end) return MultiQueryProcessingStage::QUERIES_END; - unsigned max_parser_depth = static_cast(client_context->getSettingsRef().max_parser_depth); - unsigned max_parser_backtracks = static_cast(client_context->getSettingsRef().max_parser_backtracks); + unsigned max_parser_depth = static_cast(client_context->getSettingsRef()[Setting::max_parser_depth]); + unsigned max_parser_backtracks = static_cast(client_context->getSettingsRef()[Setting::max_parser_backtracks]); // If there are only comments left until the end of file, we just // stop. The parser can't handle this situation because it always @@ -2400,9 +2414,10 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text) { this_query_end = insert_ast->end; adjustQueryEnd( - this_query_end, all_queries_end, - static_cast(client_context->getSettingsRef().max_parser_depth), - static_cast(client_context->getSettingsRef().max_parser_backtracks)); + this_query_end, + all_queries_end, + static_cast(client_context->getSettingsRef()[Setting::max_parser_depth]), + static_cast(client_context->getSettingsRef()[Setting::max_parser_backtracks])); } // Report error. diff --git a/src/Client/ClientBaseOptimizedParts.cpp b/src/Client/ClientBaseOptimizedParts.cpp index 297b8e7ce51..4222aab63b2 100644 --- a/src/Client/ClientBaseOptimizedParts.cpp +++ b/src/Client/ClientBaseOptimizedParts.cpp @@ -83,9 +83,9 @@ private: void ClientApplicationBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) { if (allow_repeated_settings) - addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value()); + cmd_settings.addToProgramOptionsAsMultitokens(options_description.main_description.value()); else - addProgramOptions(cmd_settings, options_description.main_description.value()); + cmd_settings.addToProgramOptions(options_description.main_description.value()); if (allow_merge_tree_settings) { diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 8a1c7d3988a..416bb2f0b15 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -51,6 +51,15 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_codecs; + extern const SettingsBool allow_suspicious_codecs; + extern const SettingsBool enable_deflate_qpl_codec; + extern const SettingsBool enable_zstd_qat_codec; + extern const SettingsString network_compression_method; + extern const SettingsInt64 network_zstd_compression_level; +} namespace FailPoints { @@ -791,19 +800,19 @@ void Connection::sendQuery( if (settings) { std::optional level; - std::string method = Poco::toUpper(settings->network_compression_method.toString()); + std::string method = Poco::toUpper((*settings)[Setting::network_compression_method].toString()); /// Bad custom logic if (method == "ZSTD") - level = settings->network_zstd_compression_level; + level = (*settings)[Setting::network_zstd_compression_level]; CompressionCodecFactory::instance().validateCodec( method, level, - !settings->allow_suspicious_codecs, - settings->allow_experimental_codecs, - settings->enable_deflate_qpl_codec, - settings->enable_zstd_qat_codec); + !(*settings)[Setting::allow_suspicious_codecs], + (*settings)[Setting::allow_experimental_codecs], + (*settings)[Setting::enable_deflate_qpl_codec], + (*settings)[Setting::enable_zstd_qat_codec]); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index f96546846c7..22750ac7d78 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -14,6 +14,10 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_replica_delay_for_distributed_queries; +} namespace ErrorCodes { @@ -78,7 +82,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: LOG_TRACE(log, "Table {}.{} is readonly on server {}", table_to_check->database, table_to_check->table, result.entry->getDescription()); } - const UInt64 max_allowed_delay = settings.max_replica_delay_for_distributed_queries; + const UInt64 max_allowed_delay = settings[Setting::max_replica_delay_for_distributed_queries]; if (!max_allowed_delay) { result.is_up_to_date = true; diff --git a/src/Client/ConnectionPool.cpp b/src/Client/ConnectionPool.cpp index ab8ad08826c..449834b23e2 100644 --- a/src/Client/ConnectionPool.cpp +++ b/src/Client/ConnectionPool.cpp @@ -5,6 +5,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsMilliseconds connection_pool_max_wait_ms; +} ConnectionPoolPtr ConnectionPoolFactory::get( unsigned max_connections, @@ -93,7 +97,7 @@ ConnectionPoolFactory & ConnectionPoolFactory::instance() IConnectionPool::Entry ConnectionPool::get(const DB::ConnectionTimeouts& timeouts, const DB::Settings& settings, bool force_connected) { - Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds()); + Entry entry = Base::get(settings[Setting::connection_pool_max_wait_ms].totalMilliseconds()); if (force_connected) entry->forceConnected(timeouts); diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index a5c14dc9957..a1c9ce35510 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -16,6 +16,17 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 connections_with_failover_max_tries; + extern const SettingsBool distributed_insert_skip_read_only_replicas; + extern const SettingsUInt64 distributed_replica_max_ignored_errors; + extern const SettingsBool fallback_to_stale_replicas_for_distributed_queries; + extern const SettingsLoadBalancing load_balancing; + extern const SettingsUInt64 load_balancing_first_offset; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsBool skip_unavailable_shards; +} namespace ErrorCodes { @@ -47,10 +58,10 @@ ConnectionPoolWithFailover::ConnectionPoolWithFailover( IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts & timeouts) { Settings settings; - settings.load_balancing = get_priority_load_balancing.load_balancing; - settings.load_balancing_first_offset = 0; - settings.distributed_replica_max_ignored_errors = 0; - settings.fallback_to_stale_replicas_for_distributed_queries = true; + settings[Setting::load_balancing] = get_priority_load_balancing.load_balancing; + settings[Setting::load_balancing_first_offset] = 0; + settings[Setting::distributed_replica_max_ignored_errors] = 0; + settings[Setting::fallback_to_stale_replicas_for_distributed_queries] = true; return get(timeouts, settings, /* force_connected= */ true); } @@ -68,13 +79,12 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts return tryGetEntry(pool, timeouts, fail_message, settings); }; - const size_t offset = settings.load_balancing_first_offset % nested_pools.size(); - const LoadBalancing load_balancing = settings.load_balancing; + const size_t offset = settings[Setting::load_balancing_first_offset] % nested_pools.size(); - GetPriorityFunc get_priority = get_priority_load_balancing.getPriorityFunc(load_balancing, offset, nested_pools.size()); + GetPriorityFunc get_priority = get_priority_load_balancing.getPriorityFunc(settings[Setting::load_balancing], offset, nested_pools.size()); - const UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors; - const bool fallback_to_stale_replicas = settings.fallback_to_stale_replicas_for_distributed_queries; + const UInt64 max_ignored_errors = settings[Setting::distributed_replica_max_ignored_errors]; + const bool fallback_to_stale_replicas = settings[Setting::fallback_to_stale_replicas_for_distributed_queries]; return Base::get(max_ignored_errors, fallback_to_stale_replicas, try_get_entry, get_priority); } @@ -170,15 +180,13 @@ std::vector ConnectionPoolWithFailover::g return getManyImpl(settings, pool_mode, try_get_entry, /*skip_unavailable_endpoints=*/ false, /// skip_unavailable_endpoints is used to get the min number of entries, and we need at least one /*priority_func=*/ {}, - settings.distributed_insert_skip_read_only_replicas); + settings[Setting::distributed_insert_skip_read_only_replicas]); } ConnectionPoolWithFailover::Base::GetPriorityFunc ConnectionPoolWithFailover::makeGetPriorityFunc(const Settings & settings) { - const size_t offset = settings.load_balancing_first_offset % nested_pools.size(); - const LoadBalancing load_balancing = LoadBalancing(settings.load_balancing); - - return get_priority_load_balancing.getPriorityFunc(load_balancing, offset, nested_pools.size()); + const size_t offset = settings[Setting::load_balancing_first_offset] % nested_pools.size(); + return get_priority_load_balancing.getPriorityFunc(settings[Setting::load_balancing], offset, nested_pools.size()); } std::vector ConnectionPoolWithFailover::getManyImpl( @@ -195,11 +203,11 @@ std::vector ConnectionPoolWithFailover::g "Cannot get connection from ConnectionPoolWithFailover cause nested pools are empty"); if (!skip_unavailable_endpoints.has_value()) - skip_unavailable_endpoints = settings.skip_unavailable_shards; + skip_unavailable_endpoints = settings[Setting::skip_unavailable_shards]; size_t min_entries = skip_unavailable_endpoints.value() ? 0 : 1; - size_t max_tries = settings.connections_with_failover_max_tries; + size_t max_tries = settings[Setting::connections_with_failover_max_tries]; size_t max_entries; if (pool_mode == PoolMode::GET_ALL) { @@ -212,7 +220,7 @@ std::vector ConnectionPoolWithFailover::g } else if (pool_mode == PoolMode::GET_MANY) { - max_entries = settings.max_parallel_replicas; + max_entries = settings[Setting::max_parallel_replicas]; } else { @@ -222,8 +230,8 @@ std::vector ConnectionPoolWithFailover::g if (!priority_func) priority_func = makeGetPriorityFunc(settings); - UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors.value; - bool fallback_to_stale_replicas = settings.fallback_to_stale_replicas_for_distributed_queries.value; + UInt64 max_ignored_errors = settings[Setting::distributed_replica_max_ignored_errors].value; + bool fallback_to_stale_replicas = settings[Setting::fallback_to_stale_replicas_for_distributed_queries].value; return Base::getMany(min_entries, max_entries, max_tries, max_ignored_errors, fallback_to_stale_replicas, skip_read_only_replicas, try_get_entry, priority_func); } @@ -272,7 +280,7 @@ ConnectionPoolWithFailover::getShuffledPools(const Settings & settings, GetPrior if (!priority_func) priority_func = makeGetPriorityFunc(settings); - UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors.value; + UInt64 max_ignored_errors = settings[Setting::distributed_replica_max_ignored_errors].value; return Base::getShuffledPools(max_ignored_errors, priority_func, use_slowdown_count); } diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index dd8348ea04f..b932156b2d8 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -14,6 +14,21 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_changing_replica_until_first_data_packet; + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsUInt64 connections_with_failover_max_tries; + extern const SettingsDialect dialect; + extern const SettingsBool fallback_to_stale_replicas_for_distributed_queries; + extern const SettingsUInt64 group_by_two_level_threshold; + extern const SettingsUInt64 group_by_two_level_threshold_bytes; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsUInt64 parallel_replica_offset; + extern const SettingsBool skip_unavailable_shards; +} + namespace ErrorCodes { extern const int MISMATCH_REPLICAS_DATA_SOURCES; @@ -32,15 +47,15 @@ HedgedConnections::HedgedConnections( AsyncCallback async_callback, GetPriorityForLoadBalancing::Func priority_func) : hedged_connections_factory( - pool_, - context_->getSettingsRef(), - timeouts_, - context_->getSettingsRef().connections_with_failover_max_tries.value, - context_->getSettingsRef().fallback_to_stale_replicas_for_distributed_queries.value, - context_->getSettingsRef().max_parallel_replicas.value, - context_->getSettingsRef().skip_unavailable_shards.value, - table_to_check_, - priority_func) + pool_, + context_->getSettingsRef(), + timeouts_, + context_->getSettingsRef()[Setting::connections_with_failover_max_tries].value, + context_->getSettingsRef()[Setting::fallback_to_stale_replicas_for_distributed_queries].value, + context_->getSettingsRef()[Setting::max_parallel_replicas].value, + context_->getSettingsRef()[Setting::skip_unavailable_shards].value, + table_to_check_, + priority_func) , context(std::move(context_)) , settings(context->getSettingsRef()) , throttler(throttler_) @@ -178,29 +193,29 @@ void HedgedConnections::sendQuery( Settings modified_settings = settings; /// Queries in foreign languages are transformed to ClickHouse-SQL. Ensure the setting before sending. - modified_settings.dialect = Dialect::clickhouse; - modified_settings.dialect.changed = false; + modified_settings[Setting::dialect] = Dialect::clickhouse; + modified_settings[Setting::dialect].changed = false; if (disable_two_level_aggregation) { /// Disable two-level aggregation due to version incompatibility. - modified_settings.group_by_two_level_threshold = 0; - modified_settings.group_by_two_level_threshold_bytes = 0; + modified_settings[Setting::group_by_two_level_threshold] = 0; + modified_settings[Setting::group_by_two_level_threshold_bytes] = 0; } const bool enable_offset_parallel_processing = context->canUseOffsetParallelReplicas(); if (offset_states.size() > 1 && enable_offset_parallel_processing) { - modified_settings.parallel_replicas_count = offset_states.size(); - modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset; + modified_settings[Setting::parallel_replicas_count] = offset_states.size(); + modified_settings[Setting::parallel_replica_offset] = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset; } /// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting. /// Make the analyzer being set, so it will be effectively applied on the remote server. /// In other words, the initiator always controls whether the analyzer enabled or not for /// all servers involved in the distributed query processing. - modified_settings.set("allow_experimental_analyzer", static_cast(modified_settings.allow_experimental_analyzer)); + modified_settings.set("allow_experimental_analyzer", static_cast(modified_settings[Setting::allow_experimental_analyzer])); replica.connection->sendQuery( timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {}); @@ -446,7 +461,7 @@ Packet HedgedConnections::receivePacketFromReplica(const ReplicaLocation & repli { /// If we are allowed to change replica until the first data packet, /// just restart timeout (if it hasn't expired yet). Otherwise disable changing replica with this offset. - if (settings.allow_changing_replica_until_first_data_packet && !replica.is_change_replica_timeout_expired) + if (settings[Setting::allow_changing_replica_until_first_data_packet] && !replica.is_change_replica_timeout_expired) replica.change_replica_timeout.setRelative(hedged_connections_factory.getConnectionTimeouts().receive_data_timeout); else disableChangingReplica(replica_location); diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 7595a29912b..f81eb4db959 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -23,6 +23,17 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsDialect dialect; + extern const SettingsBool input_format_defaults_for_omitted_fields; + extern const SettingsUInt64 interactive_delay; + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} namespace ErrorCodes { @@ -158,21 +169,38 @@ void LocalConnection::sendQuery( const auto & settings = context->getSettingsRef(); const char * begin = state->query.data(); const char * end = begin + state->query.size(); - const Dialect & dialect = settings.dialect; + const Dialect & dialect = settings[Setting::dialect]; std::unique_ptr parser; if (dialect == Dialect::kusto) - parser = std::make_unique(end, settings.allow_settings_after_format_in_insert); + parser = std::make_unique(end, settings[Setting::allow_settings_after_format_in_insert]); else if (dialect == Dialect::prql) - parser = std::make_unique(settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parser + = std::make_unique(settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); else - parser = std::make_unique(end, settings.allow_settings_after_format_in_insert); + parser = std::make_unique(end, settings[Setting::allow_settings_after_format_in_insert]); ASTPtr parsed_query; if (dialect == Dialect::kusto) - parsed_query = parseKQLQueryAndMovePosition(*parser, begin, end, "", /*allow_multi_statements*/false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parsed_query = parseKQLQueryAndMovePosition( + *parser, + begin, + end, + "", + /*allow_multi_statements*/ false, + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); else - parsed_query = parseQueryAndMovePosition(*parser, begin, end, "", /*allow_multi_statements*/false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parsed_query = parseQueryAndMovePosition( + *parser, + begin, + end, + "", + /*allow_multi_statements*/ false, + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); if (const auto * insert = parsed_query->as()) { @@ -180,7 +208,7 @@ void LocalConnection::sendQuery( current_format = insert->format; } - auto source = context->getInputFormat(current_format, *in, sample, context->getSettingsRef().max_insert_block_size); + auto source = context->getInputFormat(current_format, *in, sample, context->getSettingsRef()[Setting::max_insert_block_size]); Pipe pipe(source); auto columns_description = metadata_snapshot->getColumns(); @@ -227,7 +255,7 @@ void LocalConnection::sendQuery( } const auto & table_id = query_context->getInsertionTable(); - if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields) + if (query_context->getSettingsRef()[Setting::input_format_defaults_for_omitted_fields]) { if (!table_id.empty()) { @@ -255,7 +283,7 @@ void LocalConnection::sendQuery( return false; }; - executor.setCancelCallback(callback, query_context->getSettingsRef().interactive_delay / 1000); + executor.setCancelCallback(callback, query_context->getSettingsRef()[Setting::interactive_delay] / 1000); } executor.execute(); } @@ -312,7 +340,7 @@ void LocalConnection::sendCancel() bool LocalConnection::pullBlock(Block & block) { if (state->executor) - return state->executor->pull(block, query_context->getSettingsRef().interactive_delay / 1000); + return state->executor->pull(block, query_context->getSettingsRef()[Setting::interactive_delay] / 1000); return false; } @@ -466,14 +494,15 @@ bool LocalConnection::poll(size_t) bool LocalConnection::needSendProgressOrMetrics() { - if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) + if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef()[Setting::interactive_delay])) { state->after_send_progress.restart(); next_packet_type = Protocol::Server::Progress; return true; } - if (send_profile_events && (state->after_send_profile_events.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) + if (send_profile_events + && (state->after_send_profile_events.elapsedMicroseconds() >= query_context->getSettingsRef()[Setting::interactive_delay])) { sendProfileEvents(); return true; diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 244eccf1ed9..1cc6ec537c8 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -12,6 +12,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsDialect dialect; + extern const SettingsUInt64 group_by_two_level_threshold; + extern const SettingsUInt64 group_by_two_level_threshold_bytes; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsUInt64 parallel_replica_offset; + extern const SettingsSeconds receive_timeout; +} // NOLINTBEGIN(bugprone-undefined-memory-manipulation) @@ -128,8 +138,8 @@ void MultiplexedConnections::sendQuery( Settings modified_settings = settings; /// Queries in foreign languages are transformed to ClickHouse-SQL. Ensure the setting before sending. - modified_settings.dialect = Dialect::clickhouse; - modified_settings.dialect.changed = false; + modified_settings[Setting::dialect] = Dialect::clickhouse; + modified_settings[Setting::dialect].changed = false; for (auto & replica : replica_states) { @@ -139,8 +149,8 @@ void MultiplexedConnections::sendQuery( if (replica.connection->getServerRevision(timeouts) < DBMS_MIN_REVISION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD) { /// Disable two-level aggregation due to version incompatibility. - modified_settings.group_by_two_level_threshold = 0; - modified_settings.group_by_two_level_threshold_bytes = 0; + modified_settings[Setting::group_by_two_level_threshold] = 0; + modified_settings[Setting::group_by_two_level_threshold_bytes] = 0; } } @@ -154,7 +164,7 @@ void MultiplexedConnections::sendQuery( /// Make the analyzer being set, so it will be effectively applied on the remote server. /// In other words, the initiator always controls whether the analyzer enabled or not for /// all servers involved in the distributed query processing. - modified_settings.set("allow_experimental_analyzer", static_cast(modified_settings.allow_experimental_analyzer)); + modified_settings.set("allow_experimental_analyzer", static_cast(modified_settings[Setting::allow_experimental_analyzer])); const bool enable_offset_parallel_processing = context->canUseOffsetParallelReplicas(); @@ -163,12 +173,12 @@ void MultiplexedConnections::sendQuery( { if (enable_offset_parallel_processing) /// Use multiple replicas for parallel query processing. - modified_settings.parallel_replicas_count = num_replicas; + modified_settings[Setting::parallel_replicas_count] = num_replicas; for (size_t i = 0; i < num_replicas; ++i) { if (enable_offset_parallel_processing) - modified_settings.parallel_replica_offset = i; + modified_settings[Setting::parallel_replica_offset] = i; replica_states[i].connection->sendQuery( timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {}); @@ -403,7 +413,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead Poco::Net::Socket::SocketList write_list; Poco::Net::Socket::SocketList except_list; - auto timeout = settings.receive_timeout; + auto timeout = settings[Setting::receive_timeout]; int n = 0; /// EINTR loop diff --git a/src/Client/examples/CMakeLists.txt b/src/Client/examples/CMakeLists.txt index 9d788a49c59..469f7aa0852 100644 --- a/src/Client/examples/CMakeLists.txt +++ b/src/Client/examples/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable(test-connect test_connect.cpp) -target_link_libraries (test-connect PRIVATE dbms) +target_link_libraries (test-connect PRIVATE dbms clickhouse_functions) diff --git a/src/Columns/benchmarks/CMakeLists.txt b/src/Columns/benchmarks/CMakeLists.txt index 47f5dfe4c59..04c4bb88ac3 100644 --- a/src/Columns/benchmarks/CMakeLists.txt +++ b/src/Columns/benchmarks/CMakeLists.txt @@ -1,4 +1,5 @@ clickhouse_add_executable(column_insert_many_from benchmark_column_insert_many_from.cpp) target_link_libraries (column_insert_many_from PRIVATE ch_contrib::gbenchmark_all - dbms) \ No newline at end of file + dbms + clickhouse_functions) diff --git a/src/Common/DateLUT.cpp b/src/Common/DateLUT.cpp index f9bbe2fbf40..bdf64906013 100644 --- a/src/Common/DateLUT.cpp +++ b/src/Common/DateLUT.cpp @@ -13,6 +13,13 @@ #include #include +namespace DB +{ +namespace Setting +{ + extern const SettingsTimezone session_timezone; +} +} namespace { @@ -203,5 +210,5 @@ DateLUT & DateLUT::getInstance() std::string DateLUT::extractTimezoneFromContext(DB::ContextPtr query_context) { - return query_context->getSettingsRef().session_timezone.value; + return query_context->getSettingsRef()[DB::Setting::session_timezone].value; } diff --git a/src/Common/NamedCollections/NamedCollectionsFactory.cpp b/src/Common/NamedCollections/NamedCollectionsFactory.cpp index bc283d2be1a..b498684f33f 100644 --- a/src/Common/NamedCollections/NamedCollectionsFactory.cpp +++ b/src/Common/NamedCollections/NamedCollectionsFactory.cpp @@ -1,8 +1,9 @@ -#include -#include -#include #include #include +#include +#include +#include +#include namespace DB { diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp index e9f7816ce73..0d6a6af4ef0 100644 --- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp +++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp @@ -1,28 +1,36 @@ -#include -#include -#include -#include -#include -#include -#include +#include #include #include #include #include +#include #include #include #include -#include +#include #include #include -#include -#include +#include #include +#include +#include +#include +#include +#include +#include +#include namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool fsync_metadata; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} + namespace ErrorCodes { extern const int NAMED_COLLECTION_ALREADY_EXISTS; @@ -157,7 +165,7 @@ public: writeString(write_data, out); out.next(); - if (getContext()->getSettingsRef().fsync_metadata) + if (getContext()->getSettingsRef()[Setting::fsync_metadata]) out.sync(); out.close(); @@ -573,7 +581,7 @@ ASTCreateNamedCollectionQuery NamedCollectionsMetadataStorage::readCreateQuery(c const auto & settings = getContext()->getSettingsRef(); ParserCreateNamedCollectionQuery parser; - auto ast = parseQuery(parser, query, "in file " + path, 0, settings.max_parser_depth, settings.max_parser_backtracks); + auto ast = parseQuery(parser, query, "in file " + path, 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); const auto & create_query = ast->as(); return create_query; } diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index 92803af93a9..1a2876823d4 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -14,6 +14,12 @@ namespace DB { + +namespace Setting +{ + extern const SettingsFloat opentelemetry_start_trace_probability; +} + namespace OpenTelemetry { @@ -329,7 +335,7 @@ TracingContextHolder::TracingContextHolder( return; // Start the trace with some configurable probability. - std::bernoulli_distribution should_start_trace{settings_ptr->opentelemetry_start_trace_probability}; + std::bernoulli_distribution should_start_trace{(*settings_ptr)[Setting::opentelemetry_start_trace_probability]}; if (!should_start_trace(thread_local_rng)) /// skip tracing context initialization on current thread return; diff --git a/src/Common/SettingsChanges.h b/src/Common/SettingsChanges.h index 514e9a78911..9ee151c2ec3 100644 --- a/src/Common/SettingsChanges.h +++ b/src/Common/SettingsChanges.h @@ -2,6 +2,7 @@ #include +#include namespace DB { diff --git a/src/Common/benchmarks/CMakeLists.txt b/src/Common/benchmarks/CMakeLists.txt index 690ced1eb88..8b4b9d187b7 100644 --- a/src/Common/benchmarks/CMakeLists.txt +++ b/src/Common/benchmarks/CMakeLists.txt @@ -2,6 +2,7 @@ clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and target_link_libraries (integer_hash_tables_and_hashes PRIVATE ch_contrib::gbenchmark_all dbms + clickhouse_functions ch_contrib::abseil_swiss_tables ch_contrib::sparsehash ch_contrib::wyhash @@ -11,4 +12,5 @@ target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_add_executable(orc_string_dictionary orc_string_dictionary.cpp) target_link_libraries (orc_string_dictionary PRIVATE ch_contrib::gbenchmark_all - dbms) + dbms + clickhouse_functions) diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index 8383e80d09d..47aa6a664f5 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -26,24 +26,24 @@ clickhouse_add_executable (radix_sort radix_sort.cpp) target_link_libraries (radix_sort PRIVATE clickhouse_common_io clickhouse_common_config ch_contrib::pdqsort) clickhouse_add_executable (arena_with_free_lists arena_with_free_lists.cpp) -target_link_libraries (arena_with_free_lists PRIVATE dbms) +target_link_libraries (arena_with_free_lists PRIVATE dbms clickhouse_functions) clickhouse_add_executable (lru_hash_map_perf lru_hash_map_perf.cpp) -target_link_libraries (lru_hash_map_perf PRIVATE dbms) +target_link_libraries (lru_hash_map_perf PRIVATE dbms clickhouse_functions) if (OS_LINUX) clickhouse_add_executable (thread_creation_latency thread_creation_latency.cpp) - target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io clickhouse_common_config) + target_link_libraries (thread_creation_latency PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config) endif() clickhouse_add_executable (array_cache array_cache.cpp) target_link_libraries (array_cache PRIVATE clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (space_saving space_saving.cpp) -target_link_libraries (space_saving PRIVATE clickhouse_common_io clickhouse_common_config) +target_link_libraries (space_saving PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (integer_hash_tables_benchmark integer_hash_tables_benchmark.cpp) -target_link_libraries (integer_hash_tables_benchmark PRIVATE dbms ch_contrib::abseil_swiss_tables ch_contrib::sparsehash) +target_link_libraries (integer_hash_tables_benchmark PRIVATE dbms clickhouse_functions ch_contrib::abseil_swiss_tables ch_contrib::sparsehash) clickhouse_add_executable (cow_columns cow_columns.cpp) target_link_libraries (cow_columns PRIVATE clickhouse_common_io clickhouse_common_config) @@ -69,13 +69,13 @@ clickhouse_add_executable (procfs_metrics_provider_perf procfs_metrics_provider_ target_link_libraries (procfs_metrics_provider_perf PRIVATE clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (average average.cpp) -target_link_libraries (average PRIVATE clickhouse_common_io clickhouse_common_config) +target_link_libraries (average PRIVATE dbms clickhouse_common_io clickhouse_common_config clickhouse_functions) clickhouse_add_executable (shell_command_inout shell_command_inout.cpp) target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (executable_udf executable_udf.cpp) -target_link_libraries (executable_udf PRIVATE dbms) +target_link_libraries (executable_udf PRIVATE dbms clickhouse_functions) if (ENABLE_HIVE) clickhouse_add_executable (hive_metastore_client hive_metastore_client.cpp) @@ -83,7 +83,7 @@ if (ENABLE_HIVE) endif() clickhouse_add_executable (interval_tree interval_tree.cpp) -target_link_libraries (interval_tree PRIVATE dbms) +target_link_libraries (interval_tree PRIVATE dbms clickhouse_functions) if (ENABLE_SSL) clickhouse_add_executable (encrypt_decrypt encrypt_decrypt.cpp) diff --git a/src/Compression/examples/CMakeLists.txt b/src/Compression/examples/CMakeLists.txt index fee8cf89942..f686f250ae9 100644 --- a/src/Compression/examples/CMakeLists.txt +++ b/src/Compression/examples/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable (compressed_buffer compressed_buffer.cpp) -target_link_libraries (compressed_buffer PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression) +target_link_libraries (compressed_buffer PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression) diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 6242d78aee7..110c062312c 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -1,12 +1,13 @@ #pragma once +#include #include -#include -#include +#include #include #include #include -#include +#include +#include namespace boost::program_options @@ -20,13 +21,6 @@ namespace DB class ReadBuffer; class WriteBuffer; -enum class SettingsWriteFormat : uint8_t -{ - BINARY = 0, /// Part of the settings are serialized as strings, and other part as variants. This is the old behaviour. - STRINGS_WITH_FLAGS = 1, /// All settings are serialized as strings. Before each value the flag `is_important` is serialized. - DEFAULT = STRINGS_WITH_FLAGS, -}; - /** Template class to define collections of settings. * Example of usage: * @@ -96,6 +90,8 @@ public: static String valueToStringUtil(std::string_view name, const Field & value); static Field stringToValueUtil(std::string_view name, const String & str); + static std::string_view resolveName(std::string_view name); + void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const; void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT); @@ -191,8 +187,6 @@ public: MutableRange allMutable(SkipFlags skip_flags = SKIP_NONE) { return MutableRange{*this, skip_flags}; } Range allChanged() const { return all(SKIP_UNCHANGED); } Range allUnchanged() const { return all(SKIP_CHANGED); } - Range allBuiltin() const { return all(SKIP_CUSTOM); } - Range allCustom() const { return all(SKIP_BUILTIN); } Iterator begin() const { return allChanged().begin(); } Iterator end() const { return allChanged().end(); } diff --git a/src/Core/BaseSettingsFwdMacros.h b/src/Core/BaseSettingsFwdMacros.h new file mode 100644 index 00000000000..09a4397e483 --- /dev/null +++ b/src/Core/BaseSettingsFwdMacros.h @@ -0,0 +1,7 @@ +#pragma once + +#define DECLARE_SETTING_TRAIT(CLASS_NAME, TYPE) using CLASS_NAME##TYPE = SettingField##TYPE CLASS_NAME##Impl::*; + +#define DECLARE_SETTING_SUBSCRIPT_OPERATOR(CLASS_NAME, TYPE) \ + const SettingField##TYPE & operator[](CLASS_NAME##TYPE t) const; \ + SettingField##TYPE & operator[](CLASS_NAME##TYPE t); diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 7fe23f844b6..4d33633b29d 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -27,6 +27,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 http_max_multipart_form_data_size; +} namespace ErrorCodes { @@ -182,10 +186,12 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, const Settings & settings = getContext()->getSettingsRef(); - if (settings.http_max_multipart_form_data_size) + if (settings[Setting::http_max_multipart_form_data_size]) read_buffer = std::make_unique( - stream, settings.http_max_multipart_form_data_size, - /* trow_exception */ true, /* exact_limit */ std::optional(), + stream, + settings[Setting::http_max_multipart_form_data_size], + /* trow_exception */ true, + /* exact_limit */ std::optional(), "the maximum size of multipart/form-data. " "This limit can be tuned by 'http_max_multipart_form_data_size' setting"); else diff --git a/src/Core/FormatFactorySettings.cpp b/src/Core/FormatFactorySettings.cpp new file mode 100644 index 00000000000..902473a42d1 --- /dev/null +++ b/src/Core/FormatFactorySettings.cpp @@ -0,0 +1,18 @@ +#include +#include +#include + +namespace DB +{ +/* + * User-specified file format settings for File and URL engines. + */ +DECLARE_SETTINGS_TRAITS(FormatFactorySettingsTraits, LIST_OF_ALL_FORMAT_SETTINGS) + +struct FormatFactorySettingsImpl : public BaseSettings +{ +}; + +IMPLEMENT_SETTINGS_TRAITS(FormatFactorySettingsTraits, LIST_OF_ALL_FORMAT_SETTINGS) + +} diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h new file mode 100644 index 00000000000..e7749e91fbb --- /dev/null +++ b/src/Core/FormatFactorySettings.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ +struct FormatFactorySettingsImpl; +struct SettingChange; +class SettingsChanges; + +#define FORMAT_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ + M(CLASS_NAME, Bool) \ + M(CLASS_NAME, Char) \ + M(CLASS_NAME, Int64) \ + M(CLASS_NAME, UInt64) \ + M(CLASS_NAME, MsgPackUUIDRepresentation) \ + M(CLASS_NAME, SchemaInferenceMode) \ + M(CLASS_NAME, UInt64Auto) \ + M(CLASS_NAME, DateTimeInputFormat) \ + M(CLASS_NAME, DateTimeOutputFormat) \ + M(CLASS_NAME, IntervalOutputFormat) \ + M(CLASS_NAME, String) \ + M(CLASS_NAME, ParquetVersion) \ + M(CLASS_NAME, ParquetCompression) \ + M(CLASS_NAME, EscapingRule) \ + M(CLASS_NAME, ArrowCompression) \ + M(CLASS_NAME, CapnProtoEnumComparingMode) \ + M(CLASS_NAME, DateTimeOverflowBehavior) \ + M(CLASS_NAME, IdentifierQuotingStyle) + +FORMAT_SETTINGS_SUPPORTED_TYPES(FormatFactorySettings, DECLARE_SETTING_TRAIT) + +struct FormatFactorySettings +{ + FormatFactorySettings(); + ~FormatFactorySettings(); + + FORMAT_SETTINGS_SUPPORTED_TYPES(FormatFactorySettings, DECLARE_SETTING_SUBSCRIPT_OPERATOR) + + /// General API as needed + bool tryGet(std::string_view name, Field & value) const; + Field get(std::string_view name) const; + void set(std::string_view name, const Field & value); + bool has(std::string_view name) const; + void applyChange(const SettingChange & change); + void applyChanges(const SettingsChanges & changes); + +private: + std::unique_ptr impl; +}; + +} diff --git a/src/Core/FormatFactorySettingsDeclaration.h b/src/Core/FormatFactorySettingsDeclaration.h new file mode 100644 index 00000000000..3ef2e174cbe --- /dev/null +++ b/src/Core/FormatFactorySettingsDeclaration.h @@ -0,0 +1,275 @@ +#pragma once + +#include + +/// This header exists so we can share it between Settings.cpp, FormatFactorySettings.cpp and other storage settings + +// clang-format off +#if defined(__CLION_IDE__) +/// CLion freezes for a minute every time is processes this +#define FORMAT_FACTORY_SETTINGS(M, ALIAS) +#define OBSOLETE_FORMAT_SETTINGS(M, ALIAS) +#else + +#define FORMAT_FACTORY_SETTINGS(M, ALIAS) \ + M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \ + M(Bool, format_csv_allow_single_quotes, false, "If it is set to true, allow strings in single quotes.", 0) \ + M(Bool, format_csv_allow_double_quotes, true, "If it is set to true, allow strings in double quotes.", 0) \ + M(Bool, output_format_csv_serialize_tuple_into_separate_columns, true, "If it set to true, then Tuples in CSV format are serialized as separate columns (that is, their nesting in the tuple is lost)", 0) \ + M(Bool, input_format_csv_deserialize_separate_columns_into_tuple, true, "If it set to true, then separate columns written in CSV format can be deserialized to Tuple column.", 0) \ + M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \ + M(Bool, input_format_csv_allow_cr_end_of_line, false, "If it is set true, \\r will be allowed at end of line not followed by \\n", 0) \ + M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices", 0) \ + M(Bool, input_format_csv_arrays_as_nested_csv, false, R"(When reading Array from CSV, expect that its elements were serialized in nested CSV and then put into string. Example: "[""Hello"", ""world"", ""42"""" TV""]". Braces around array can be omitted.)", 0) \ + M(Bool, input_format_skip_unknown_fields, true, "Skip columns with unknown names from input data (it works for JSONEachRow, -WithNames, -WithNamesAndTypes and TSKV formats).", 0) \ + M(Bool, input_format_with_names_use_header, true, "For -WithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ + M(Bool, input_format_with_types_use_header, true, "For -WithNamesAndTypes input formats this controls whether format parser should check if data types from the input match data types from the header.", 0) \ + M(Bool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ + M(Bool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, -WithNames, -WithNamesAndTypes formats).", IMPORTANT) \ + M(Bool, input_format_csv_empty_as_default, true, "Treat empty fields in CSV input as default values.", 0) \ + M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ + M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices.", 0) \ + M(Bool, input_format_null_as_default, true, "Initialize null fields with default values if the data type of this field is not nullable and it is supported by the input format", 0) \ + M(Bool, input_format_force_null_for_omitted_fields, false, "Force initialize omitted fields with null values", 0) \ + M(Bool, input_format_arrow_case_insensitive_column_matching, false, "Ignore case when matching Arrow columns with CH columns.", 0) \ + M(Int64, input_format_orc_row_batch_size, 100'000, "Batch size when reading ORC stripes.", 0) \ + M(Bool, input_format_orc_case_insensitive_column_matching, false, "Ignore case when matching ORC columns with CH columns.", 0) \ + M(Bool, input_format_parquet_case_insensitive_column_matching, false, "Ignore case when matching Parquet columns with CH columns.", 0) \ + M(Bool, input_format_parquet_preserve_order, false, "Avoid reordering rows when reading from Parquet files. Usually makes it much slower.", 0) \ + M(Bool, input_format_parquet_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and min/max statistics in the Parquet metadata.", 0) \ + M(Bool, input_format_parquet_use_native_reader, false, "When reading Parquet files, to use native reader instead of arrow reader.", 0) \ + M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \ + M(Bool, input_format_orc_allow_missing_columns, true, "Allow missing columns while reading ORC input formats", 0) \ + M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \ + M(Bool, input_format_orc_filter_push_down, true, "When reading ORC files, skip whole stripes or row groups based on the WHERE/PREWHERE expressions, min/max statistics or bloom filter in the ORC metadata.", 0) \ + M(String, input_format_orc_reader_time_zone_name, "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT.", 0) \ + M(Bool, input_format_parquet_allow_missing_columns, true, "Allow missing columns while reading Parquet input formats", 0) \ + M(UInt64, input_format_parquet_local_file_min_bytes_for_seek, 8192, "Min bytes required for local read (file) to do seek, instead of read with ignore in Parquet input format", 0) \ + M(Bool, input_format_arrow_allow_missing_columns, true, "Allow missing columns while reading Arrow input formats", 0) \ + M(Char, input_format_hive_text_fields_delimiter, '\x01', "Delimiter between fields in Hive Text File", 0) \ + M(Char, input_format_hive_text_collection_items_delimiter, '\x02', "Delimiter between collection(array or map) items in Hive Text File", 0) \ + M(Char, input_format_hive_text_map_keys_delimiter, '\x03', "Delimiter between a pair of map key/values in Hive Text File", 0) \ + M(Bool, input_format_hive_text_allow_variable_number_of_columns, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values", 0) \ + M(UInt64, input_format_msgpack_number_of_columns, 0, "The number of columns in inserted MsgPack data. Used for automatic schema inference from data.", 0) \ + M(MsgPackUUIDRepresentation, output_format_msgpack_uuid_representation, FormatSettings::MsgPackUUIDRepresentation::EXT, "The way how to output UUID in MsgPack format.", 0) \ + M(UInt64, input_format_max_rows_to_read_for_schema_inference, 25000, "The maximum rows of data to read for automatic schema inference", 0) \ + M(UInt64, input_format_max_bytes_to_read_for_schema_inference, 32 * 1024 * 1024, "The maximum bytes of data to read for automatic schema inference", 0) \ + M(Bool, input_format_csv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in CSV format", 0) \ + M(Bool, input_format_csv_try_infer_numbers_from_strings, false, "Try to infer numbers from string fields while schema inference in CSV format", 0) \ + M(Bool, input_format_csv_try_infer_strings_from_quoted_tuples, true, "Interpret quoted tuples in the input data as a value of type String.", 0) \ + M(Bool, input_format_tsv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in TSV format", 0) \ + M(Bool, input_format_csv_detect_header, true, "Automatically detect header with names and types in CSV format", 0) \ + M(Bool, input_format_csv_allow_whitespace_or_tab_as_delimiter, false, "Allow to use spaces and tabs(\\t) as field delimiter in the CSV strings", 0) \ + M(Bool, input_format_csv_trim_whitespaces, true, "Trims spaces and tabs (\\t) characters at the beginning and end in CSV strings", 0) \ + M(Bool, input_format_csv_use_default_on_bad_values, false, "Allow to set default value to column when CSV field deserialization failed on bad value", 0) \ + M(Bool, input_format_csv_allow_variable_number_of_columns, false, "Ignore extra columns in CSV input (if file has more columns than expected) and treat missing fields in CSV input as default values", 0) \ + M(Bool, input_format_tsv_allow_variable_number_of_columns, false, "Ignore extra columns in TSV input (if file has more columns than expected) and treat missing fields in TSV input as default values", 0) \ + M(Bool, input_format_custom_allow_variable_number_of_columns, false, "Ignore extra columns in CustomSeparated input (if file has more columns than expected) and treat missing fields in CustomSeparated input as default values", 0) \ + M(Bool, input_format_json_compact_allow_variable_number_of_columns, false, "Ignore extra columns in JSONCompact(EachRow) input (if file has more columns than expected) and treat missing fields in JSONCompact(EachRow) input as default values", 0) \ + M(Bool, input_format_tsv_detect_header, true, "Automatically detect header with names and types in TSV format", 0) \ + M(Bool, input_format_custom_detect_header, true, "Automatically detect header with names and types in CustomSeparated format", 0) \ + M(Bool, input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format Parquet", 0) \ + M(UInt64, input_format_parquet_max_block_size, DEFAULT_BLOCK_SIZE, "Max block size for parquet reader.", 0) \ + M(UInt64, input_format_parquet_prefer_block_bytes, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader", 0) \ + M(Bool, input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format Protobuf", 0) \ + M(Bool, input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format CapnProto", 0) \ + M(Bool, input_format_orc_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format ORC", 0) \ + M(Bool, input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format Arrow", 0) \ + M(String, column_names_for_schema_inference, "", "The list of column names to use in schema inference for formats without column names. The format: 'column1,column2,column3,...'", 0) \ + M(String, schema_inference_hints, "", "The list of column names and types to use in schema inference for formats without column names. The format: 'column_name1 column_type1, column_name2 column_type2, ...'", 0) \ + M(SchemaInferenceMode, schema_inference_mode, "default", "Mode of schema inference. 'default' - assume that all files have the same schema and schema can be inferred from any file, 'union' - files can have different schemas and the resulting schema should be the a union of schemas of all files", 0) \ + M(UInt64Auto, schema_inference_make_columns_nullable, 1, "If set to true, all inferred types will be Nullable in schema inference. When set to false, no columns will be converted to Nullable. When set to 'auto', ClickHouse will use information about nullability from the data.", 0) \ + M(Bool, input_format_json_read_bools_as_numbers, true, "Allow to parse bools as numbers in JSON input formats", 0) \ + M(Bool, input_format_json_read_bools_as_strings, true, "Allow to parse bools as strings in JSON input formats", 0) \ + M(Bool, input_format_json_try_infer_numbers_from_strings, false, "Try to infer numbers from string fields while schema inference", 0) \ + M(Bool, input_format_json_validate_types_from_metadata, true, "For JSON/JSONCompact/JSONColumnsWithMetadata input formats this controls whether format parser should check if data types from input metadata match data types of the corresponding columns from the table", 0) \ + M(Bool, input_format_json_read_numbers_as_strings, true, "Allow to parse numbers as strings in JSON input formats", 0) \ + M(Bool, input_format_json_read_objects_as_strings, true, "Allow to parse JSON objects as strings in JSON input formats", 0) \ + M(Bool, input_format_json_read_arrays_as_strings, true, "Allow to parse JSON arrays as strings in JSON input formats", 0) \ + M(Bool, input_format_json_try_infer_named_tuples_from_objects, true, "Try to infer named tuples from JSON objects in JSON input formats", 0) \ + M(Bool, input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects, false, "Use String type instead of an exception in case of ambiguous paths in JSON objects during named tuples inference", 0) \ + M(Bool, input_format_json_infer_incomplete_types_as_strings, true, "Use type String for keys that contains only Nulls or empty objects/arrays during schema inference in JSON input formats", 0) \ + M(Bool, input_format_json_named_tuples_as_objects, true, "Deserialize named tuple columns as JSON objects", 0) \ + M(Bool, input_format_json_ignore_unknown_keys_in_named_tuple, true, "Ignore unknown keys in json object for named tuples", 0) \ + M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \ + M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \ + M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \ + M(Bool, input_format_try_infer_variants, false, "Try to infer the Variant type in text formats when there is more than one possible type for column/array elements", 0) \ + M(Bool, type_json_skip_duplicated_paths, false, "When enabled, during parsing JSON object into JSON type duplicated paths will be ignored and only the first one will be inserted instead of an exception", 0) \ + M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \ + M(Bool, input_format_json_empty_as_default, false, "Treat empty fields in JSON input as default values.", 0) \ + M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \ + M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \ + M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \ + M(Bool, input_format_try_infer_datetimes_only_datetime64, false, "When input_format_try_infer_datetimes is enabled, infer only DateTime64 but not DateTime types", 0) \ + M(Bool, input_format_try_infer_exponent_floats, false, "Try to infer floats in exponential notation while schema inference in text formats (except JSON, where exponent numbers are always inferred)", 0) \ + M(Bool, output_format_markdown_escape_special_characters, false, "Escape special characters in Markdown", 0) \ + M(Bool, input_format_protobuf_flatten_google_wrappers, false, "Enable Google wrappers for regular non-nested columns, e.g. google.protobuf.StringValue 'str' for String column 'str'. For Nullable columns empty wrappers are recognized as defaults, and missing as nulls", 0) \ + M(Bool, output_format_protobuf_nullables_with_google_wrappers, false, "When serializing Nullable columns with Google wrappers, serialize default values as empty wrappers. If turned off, default and null values are not serialized", 0) \ + M(UInt64, input_format_csv_skip_first_lines, 0, "Skip specified number of lines at the beginning of data in CSV format", 0) \ + M(UInt64, input_format_tsv_skip_first_lines, 0, "Skip specified number of lines at the beginning of data in TSV format", 0) \ + M(Bool, input_format_csv_skip_trailing_empty_lines, false, "Skip trailing empty lines in CSV format", 0) \ + M(Bool, input_format_tsv_skip_trailing_empty_lines, false, "Skip trailing empty lines in TSV format", 0) \ + M(Bool, input_format_custom_skip_trailing_empty_lines, false, "Skip trailing empty lines in CustomSeparated format", 0) \ + M(Bool, input_format_tsv_crlf_end_of_line, false, "If it is set true, file function will read TSV format with \\r\\n instead of \\n.", 0) \ + \ + M(Bool, input_format_native_allow_types_conversion, true, "Allow data types conversion in Native input format", 0) \ + M(Bool, input_format_native_decode_types_in_binary_format, false, "Read data types in binary format instead of type names in Native input format", 0) \ + M(Bool, output_format_native_encode_types_in_binary_format, false, "Write data types in binary format instead of type names in Native output format", 0) \ + \ + M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ + M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ + M(IntervalOutputFormat, interval_output_format, FormatSettings::IntervalOutputFormat::Numeric, "Textual representation of Interval. Possible values: 'kusto', 'numeric'.", 0) \ + \ + M(Bool, input_format_ipv4_default_on_conversion_error, false, "Deserialization of IPv4 will use default values instead of throwing exception on conversion error.", 0) \ + M(Bool, input_format_ipv6_default_on_conversion_error, false, "Deserialization of IPV6 will use default values instead of throwing exception on conversion error.", 0) \ + M(String, bool_true_representation, "true", "Text to represent bool value in TSV/CSV formats.", 0) \ + M(String, bool_false_representation, "false", "Text to represent bool value in TSV/CSV formats.", 0) \ + \ + M(Bool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ + M(Bool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ + M(Bool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ + M(Bool, input_format_avro_allow_missing_fields, false, "For Avro/AvroConfluent format: when field is not found in schema use default value instead of error", 0) \ + /** This setting is obsolete and do nothing, left for compatibility reasons. */ \ + M(Bool, input_format_avro_null_as_default, false, "For Avro/AvroConfluent format: insert default in case of null and non Nullable column", 0) \ + M(UInt64, format_binary_max_string_size, 1_GiB, "The maximum allowed size for String in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \ + M(UInt64, format_binary_max_array_size, 1_GiB, "The maximum allowed size for Array in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \ + M(Bool, input_format_binary_decode_types_in_binary_format, false, "Read data types in binary format instead of type names in RowBinaryWithNamesAndTypes input format", 0) \ + M(Bool, output_format_binary_encode_types_in_binary_format, false, "Write data types in binary format instead of type names in RowBinaryWithNamesAndTypes output format ", 0) \ + M(URI, format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \ + \ + M(Bool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \ + M(Bool, output_format_json_quote_denormals, false, "Enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format.", 0) \ + M(Bool, output_format_json_quote_decimals, false, "Controls quoting of decimals in JSON output format.", 0) \ + M(Bool, output_format_json_quote_64bit_floats, false, "Controls quoting of 64-bit float numbers in JSON output format.", 0) \ + \ + M(Bool, output_format_json_escape_forward_slashes, true, "Controls escaping forward slashes for string outputs in JSON output format. This is intended for compatibility with JavaScript. Don't confuse with backslashes that are always escaped.", 0) \ + M(Bool, output_format_json_named_tuples_as_objects, true, "Serialize named tuple columns as JSON objects.", 0) \ + M(Bool, output_format_json_skip_null_value_in_named_tuples, false, "Skip key value pairs with null value when serialize named tuple columns as JSON objects. It is only valid when output_format_json_named_tuples_as_objects is true.", 0) \ + M(Bool, output_format_json_array_of_rows, false, "Output a JSON array of all rows in JSONEachRow(Compact) format.", 0) \ + M(Bool, output_format_json_validate_utf8, false, "Validate UTF-8 sequences in JSON output formats, doesn't impact formats JSON/JSONCompact/JSONColumnsWithMetadata, they always validate utf8", 0) \ + \ + M(String, format_json_object_each_row_column_for_object_name, "", "The name of column that will be used as object names in JSONObjectEachRow format. Column type should be String", 0) \ + \ + M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ + M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ + M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \ + M(UInt64, output_format_pretty_max_value_width_apply_for_single_value, false, "Only cut values (see the `output_format_pretty_max_value_width` setting) when it is not a single value in a block. Otherwise output it entirely, which is useful for the `SHOW CREATE TABLE` query.", 0) \ + M(UInt64Auto, output_format_pretty_color, "auto", "Use ANSI escape sequences in Pretty formats. 0 - disabled, 1 - enabled, 'auto' - enabled if a terminal.", 0) \ + M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \ + M(UInt64, output_format_pretty_display_footer_column_names, true, "Display column names in the footer if there are 999 or more rows.", 0) \ + M(UInt64, output_format_pretty_display_footer_column_names_min_rows, 50, "Sets the minimum threshold value of rows for which to enable displaying column names in the footer. 50 (default)", 0) \ + M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \ + M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \ + M(Bool, output_format_parquet_string_as_string, true, "Use Parquet String type instead of Binary for String columns.", 0) \ + M(Bool, output_format_parquet_fixed_string_as_fixed_byte_array, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary for FixedString columns.", 0) \ + M(ParquetVersion, output_format_parquet_version, "2.latest", "Parquet format version for output format. Supported versions: 1.0, 2.4, 2.6 and 2.latest (default)", 0) \ + M(ParquetCompression, output_format_parquet_compression_method, "zstd", "Compression method for Parquet output format. Supported codecs: snappy, lz4, brotli, zstd, gzip, none (uncompressed)", 0) \ + M(Bool, output_format_parquet_compliant_nested_types, true, "In parquet file schema, use name 'element' instead of 'item' for list elements. This is a historical artifact of Arrow library implementation. Generally increases compatibility, except perhaps with some old versions of Arrow.", 0) \ + M(Bool, output_format_parquet_use_custom_encoder, true, "Use a faster Parquet encoder implementation.", 0) \ + M(Bool, output_format_parquet_parallel_encoding, true, "Do Parquet encoding in multiple threads. Requires output_format_parquet_use_custom_encoder.", 0) \ + M(UInt64, output_format_parquet_data_page_size, 1024 * 1024, "Target page size in bytes, before compression.", 0) \ + M(UInt64, output_format_parquet_batch_size, 1024, "Check page size every this many rows. Consider decreasing if you have columns with average values size above a few KBs.", 0) \ + M(Bool, output_format_parquet_write_page_index, true, "Add a possibility to write page index into parquet files.", 0) \ + M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy', 'zstd'.", 0) \ + M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ + M(String, output_format_avro_string_column_pattern, "", "For Avro format: regexp of String columns to select as AVRO string.", 0) \ + M(UInt64, output_format_avro_rows_in_file, 1, "Max rows in a file (if permitted by storage)", 0) \ + M(Bool, output_format_tsv_crlf_end_of_line, false, "If it is set true, end of line in TSV format will be \\r\\n instead of \\n.", 0) \ + M(String, format_csv_null_representation, "\\N", "Custom NULL representation in CSV format", 0) \ + M(String, format_tsv_null_representation, "\\N", "Custom NULL representation in TSV format", 0) \ + M(Bool, output_format_decimal_trailing_zeros, false, "Output trailing zeros when printing Decimal values. E.g. 1.230000 instead of 1.23.", 0) \ + \ + M(UInt64, input_format_allow_errors_num, 0, "Maximum absolute amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ + M(Float, input_format_allow_errors_ratio, 0, "Maximum relative amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ + M(String, input_format_record_errors_file_path, "", "Path of the file used to record errors while reading text formats (CSV, TSV).", 0) \ + M(String, errors_output_format, "CSV", "Method to write Errors to text output.", 0) \ + \ + M(String, format_schema, "", "Schema identifier (used by schema-based formats)", 0) \ + M(String, format_template_resultset, "", "Path to file which contains format string for result set (for Template format)", 0) \ + M(String, format_template_row, "", "Path to file which contains format string for rows (for Template format)", 0) \ + M(String, format_template_row_format, "", "Format string for rows (for Template format)", 0) \ + M(String, format_template_resultset_format, "", "Format string for result set (for Template format)", 0) \ + M(String, format_template_rows_between_delimiter, "\n", "Delimiter between rows (for Template format)", 0) \ + \ + M(EscapingRule, format_custom_escaping_rule, "Escaped", "Field escaping rule (for CustomSeparated format)", 0) \ + M(String, format_custom_field_delimiter, "\t", "Delimiter between fields (for CustomSeparated format)", 0) \ + M(String, format_custom_row_before_delimiter, "", "Delimiter before field of the first column (for CustomSeparated format)", 0) \ + M(String, format_custom_row_after_delimiter, "\n", "Delimiter after field of the last column (for CustomSeparated format)", 0) \ + M(String, format_custom_row_between_delimiter, "", "Delimiter between rows (for CustomSeparated format)", 0) \ + M(String, format_custom_result_before_delimiter, "", "Prefix before result set (for CustomSeparated format)", 0) \ + M(String, format_custom_result_after_delimiter, "", "Suffix after result set (for CustomSeparated format)", 0) \ + \ + M(String, format_regexp, "", "Regular expression (for Regexp format)", 0) \ + M(EscapingRule, format_regexp_escaping_rule, "Raw", "Field escaping rule (for Regexp format)", 0) \ + M(Bool, format_regexp_skip_unmatched, false, "Skip lines unmatched by regular expression (for Regexp format)", 0) \ + \ + M(Bool, output_format_enable_streaming, false, "Enable streaming in output formats that support it.", 0) \ + M(Bool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ + M(Bool, output_format_pretty_row_numbers, true, "Add row numbers before each row for pretty output format", 0) \ + M(Bool, output_format_pretty_highlight_digit_groups, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline.", 0) \ + M(UInt64, output_format_pretty_single_large_number_tip_threshold, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)", 0) \ + M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \ + \ + M(Bool, exact_rows_before_limit, false, "When enabled, ClickHouse will provide exact value for rows_before_limit_at_least statistic, but with the cost that the data before limit will have to be read completely", 0) \ + M(Bool, rows_before_aggregation, false, "When enabled, ClickHouse will provide exact value for rows_before_aggregation statistic, represents the number of rows read before aggregation", 0) \ + M(UInt64, cross_to_inner_join_rewrite, 1, "Use inner join instead of comma/cross join if there are joining expressions in the WHERE section. Values: 0 - no rewrite, 1 - apply if possible for comma/cross, 2 - force rewrite all comma joins, cross - if possible", 0) \ + \ + M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \ + M(Bool, output_format_arrow_use_signed_indexes_for_dictionary, true, "Use signed integers for dictionary indexes in Arrow format", 0) \ + M(Bool, output_format_arrow_use_64_bit_indexes_for_dictionary, false, "Always use 64 bit integers for dictionary indexes in Arrow format", 0) \ + M(Bool, output_format_arrow_string_as_string, true, "Use Arrow String type instead of Binary for String columns", 0) \ + M(Bool, output_format_arrow_fixed_string_as_fixed_byte_array, true, "Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns.", 0) \ + M(ArrowCompression, output_format_arrow_compression_method, "lz4_frame", "Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed)", 0) \ + \ + M(Bool, output_format_orc_string_as_string, true, "Use ORC String type instead of Binary for String columns", 0) \ + M(ORCCompression, output_format_orc_compression_method, "zstd", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \ + M(UInt64, output_format_orc_row_index_stride, 10'000, "Target row index stride in ORC output format", 0) \ + M(Double, output_format_orc_dictionary_key_size_threshold, 0.0, "For a string column in ORC output format, if the number of distinct values is greater than this fraction of the total number of non-null rows, turn off dictionary encoding. Otherwise dictionary encoding is enabled", 0) \ + \ + M(CapnProtoEnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::CapnProtoEnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \ + \ + M(Bool, format_capn_proto_use_autogenerated_schema, true, "Use autogenerated CapnProto schema when format_schema is not set", 0) \ + M(Bool, format_protobuf_use_autogenerated_schema, true, "Use autogenerated Protobuf when format_schema is not set", 0) \ + M(String, output_format_schema, "", "The path to the file where the automatically generated schema will be saved", 0) \ + \ + M(String, input_format_mysql_dump_table_name, "", "Name of the table in MySQL dump from which to read data", 0) \ + M(Bool, input_format_mysql_dump_map_column_names, true, "Match columns from table in MySQL dump and columns from ClickHouse table by names", 0) \ + \ + M(UInt64, output_format_sql_insert_max_batch_size, DEFAULT_BLOCK_SIZE, "The maximum number of rows in one INSERT statement.", 0) \ + M(String, output_format_sql_insert_table_name, "table", "The name of table in the output INSERT query", 0) \ + M(Bool, output_format_sql_insert_include_column_names, true, "Include column names in INSERT query", 0) \ + M(Bool, output_format_sql_insert_use_replace, false, "Use REPLACE statement instead of INSERT", 0) \ + M(Bool, output_format_sql_insert_quote_names, true, "Quote column names with '`' characters", 0) \ + \ + M(Bool, output_format_values_escape_quote_with_quote, false, "If true escape ' with '', otherwise quoted with \\'", 0) \ + \ + M(Bool, output_format_bson_string_as_string, false, "Use BSON String type instead of Binary for String columns.", 0) \ + M(Bool, input_format_bson_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format BSON.", 0) \ + \ + M(Bool, format_display_secrets_in_show_and_select, false, "Do not hide secrets in SHOW and SELECT queries.", IMPORTANT) \ + M(Bool, regexp_dict_allow_hyperscan, true, "Allow regexp_tree dictionary using Hyperscan library.", 0) \ + M(Bool, regexp_dict_flag_case_insensitive, false, "Use case-insensitive matching for a regexp_tree dictionary. Can be overridden in individual expressions with (?i) and (?-i).", 0) \ + M(Bool, regexp_dict_flag_dotall, false, "Allow '.' to match newline characters for a regexp_tree dictionary.", 0) \ + \ + M(Bool, dictionary_use_async_executor, false, "Execute a pipeline for reading dictionary source in several threads. It's supported only by dictionaries with local CLICKHOUSE source.", 0) \ + M(Bool, precise_float_parsing, false, "Prefer more precise (but slower) float parsing algorithm", 0) \ + M(DateTimeOverflowBehavior, date_time_overflow_behavior, "ignore", "Overflow mode for Date, Date32, DateTime, DateTime64 types. Possible values: 'ignore', 'throw', 'saturate'.", 0) \ + M(Bool, validate_experimental_and_suspicious_types_inside_nested_types, true, "Validate usage of experimental and suspicious types inside nested types like Array/Map/Tuple", 0) \ + \ + M(Bool, output_format_always_quote_identifiers, false, "Always quote identifiers", 0) \ + M(IdentifierQuotingStyle, output_format_identifier_quoting_style, IdentifierQuotingStyle::Backticks, "Set the quoting style for identifiers", 0) \ + +// End of FORMAT_FACTORY_SETTINGS + +#define OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \ + /** Obsolete format settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + MAKE_OBSOLETE(M, Bool, input_format_arrow_import_nested, false) \ + MAKE_OBSOLETE(M, Bool, input_format_parquet_import_nested, false) \ + MAKE_OBSOLETE(M, Bool, input_format_orc_import_nested, false) \ + +#endif // __CLION_IDE__ + +#define LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS) \ + FORMAT_FACTORY_SETTINGS(M, ALIAS) \ + OBSOLETE_FORMAT_SETTINGS(M, ALIAS) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 45bd2b9eb42..4518d78657c 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -1,10 +1,25 @@ -#include "Settings.h" - -#include -#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include + +#include +#include +#include + #include namespace DB @@ -18,12 +33,1090 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +/** List of settings: type, name, default value, description, flags + * + * This looks rather inconvenient. It is done that way to avoid repeating settings in different places. + * Note: as an alternative, we could implement settings to be completely dynamic in form of map: String -> Field, + * but we are not going to do it, because settings is used everywhere as static struct fields. + * + * `flags` can be either 0 or IMPORTANT. + * A setting is "IMPORTANT" if it affects the results of queries and can't be ignored by older versions. + * + * When adding new or changing existing settings add them to settings changes history in SettingsChangesHistory.h + * for tracking settings changes in different versions and for special `compatibility` setting to work correctly. + */ + +// clang-format off +#if defined(__CLION_IDE__) +/// CLion freezes for a minute every time is processes this +#define COMMON_SETTINGS(M, ALIAS) +#define OBSOLETE_SETTINGS(M, ALIAS) +#else +#define COMMON_SETTINGS(M, ALIAS) \ + M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\ + M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \ + M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \ + M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \ + M(UInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ + M(UInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ + M(UInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ + M(UInt64, min_insert_block_size_rows_for_materialized_views, 0, "Like min_insert_block_size_rows, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_rows)", 0) \ + M(UInt64, min_insert_block_size_bytes_for_materialized_views, 0, "Like min_insert_block_size_bytes, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_bytes)", 0) \ + M(UInt64, min_external_table_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough.", 0) \ + M(UInt64, min_external_table_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough.", 0) \ + M(UInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algorithm supports it). 0 means unlimited.", 0) \ + M(UInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. Values 0 or 1 means that INSERT SELECT is not run in parallel. Higher values will lead to higher memory usage. Parallel INSERT SELECT has effect only if the SELECT part is run on parallel, see 'max_threads' setting.", 0) \ + M(UInt64, max_insert_delayed_streams_for_parallel_write, 0, "The maximum number of streams (columns) to delay final part flush. Default - auto (1000 in case of underlying storage supports parallel write, for example S3 and disabled otherwise)", 0) \ + M(MaxThreads, max_final_threads, 0, "The maximum number of threads to read from table with FINAL.", 0) \ + M(UInt64, max_threads_for_indexes, 0, "The maximum number of threads process indices.", 0) \ + M(MaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ + M(Bool, use_concurrency_control, true, "Respect the server's concurrency control (see the `concurrent_threads_soft_limit_num` and `concurrent_threads_soft_limit_ratio_to_cores` global server settings). If disabled, it allows using a larger number of threads even if the server is overloaded (not recommended for normal usage, and needed mostly for tests).", 0) \ + M(MaxThreads, max_download_threads, 4, "The maximum number of threads to download data (e.g. for URL engine).", 0) \ + M(MaxThreads, max_parsing_threads, 0, "The maximum number of threads to parse data in input formats that support parallel parsing. By default, it is determined automatically", 0) \ + M(UInt64, max_download_buffer_size, 10*1024*1024, "The maximal size of buffer for parallel downloading (e.g. for URL engine) per each thread.", 0) \ + M(UInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ + M(UInt64, max_read_buffer_size_local_fs, 128*1024, "The maximum size of the buffer to read from local filesystem. If set to 0 then max_read_buffer_size will be used.", 0) \ + M(UInt64, max_read_buffer_size_remote_fs, 0, "The maximum size of the buffer to read from remote filesystem. If set to 0 then max_read_buffer_size will be used.", 0) \ + M(UInt64, max_distributed_connections, 1024, "The maximum number of connections for distributed processing of one query (should be greater than max_threads).", 0) \ + M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "The maximum number of bytes of a query string parsed by the SQL parser. Data in the VALUES clause of INSERT queries is processed by a separate stream parser (that consumes O(1) RAM) and not affected by this restriction.", 0) \ + M(UInt64, interactive_delay, 100000, "The interval in microseconds to check if the request is cancelled, and to send progress info.", 0) \ + M(Seconds, connect_timeout, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connection timeout if there are no replicas.", 0) \ + M(Milliseconds, handshake_timeout_ms, 10000, "Timeout for receiving HELLO packet from replicas.", 0) \ + M(Milliseconds, connect_timeout_with_failover_ms, 1000, "Connection timeout for selecting first healthy replica.", 0) \ + M(Milliseconds, connect_timeout_with_failover_secure_ms, 1000, "Connection timeout for selecting first healthy replica (for secure connections).", 0) \ + M(Seconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "Timeout for receiving data from network, in seconds. If no bytes were received in this interval, exception is thrown. If you set this setting on client, the 'send_timeout' for the socket will be also set on the corresponding connection end on the server.", 0) \ + M(Seconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "Timeout for sending data to network, in seconds. If client needs to sent some data, but it did not able to send any bytes in this interval, exception is thrown. If you set this setting on client, the 'receive_timeout' for the socket will be also set on the corresponding connection end on the server.", 0) \ + M(Seconds, tcp_keep_alive_timeout, DEFAULT_TCP_KEEP_ALIVE_TIMEOUT /* less than DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC */, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \ + M(Milliseconds, hedged_connection_timeout_ms, 50, "Connection timeout for establishing connection with replica for Hedged requests", 0) \ + M(Milliseconds, receive_data_timeout_ms, 2000, "Connection timeout for receiving first packet of data or packet with positive progress from replica", 0) \ + M(Bool, use_hedged_requests, true, "Use hedged requests for distributed queries", 0) \ + M(Bool, allow_changing_replica_until_first_data_packet, false, "Allow HedgedConnections to change replica until receiving first data packet", 0) \ + M(Milliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.", 0) \ + M(Milliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \ + M(Milliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \ + M(Milliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \ + M(Milliseconds, rabbitmq_max_wait_ms, 5000, "The wait time for reading from RabbitMQ before retry.", 0) \ + M(UInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \ + M(UInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \ + M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \ + M(UInt64, connections_with_failover_max_tries, 3, "The maximum number of attempts to connect to replicas.", 0) \ + M(UInt64, s3_strict_upload_part_size, S3::DEFAULT_STRICT_UPLOAD_PART_SIZE, "The exact size of part to upload during multipart upload to S3 (some implementations does not supports variable size parts).", 0) \ + M(UInt64, azure_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to Azure blob storage.", 0) \ + M(UInt64, azure_max_blocks_in_multipart_upload, 50000, "Maximum number of blocks in multipart upload for Azure.", 0) \ + M(UInt64, s3_min_upload_part_size, S3::DEFAULT_MIN_UPLOAD_PART_SIZE, "The minimum size of part to upload during multipart upload to S3.", 0) \ + M(UInt64, s3_max_upload_part_size, S3::DEFAULT_MAX_UPLOAD_PART_SIZE, "The maximum size of part to upload during multipart upload to S3.", 0) \ + M(UInt64, azure_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage.", 0) \ + M(UInt64, azure_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage.", 0) \ + M(UInt64, s3_upload_part_size_multiply_factor, S3::DEFAULT_UPLOAD_PART_SIZE_MULTIPLY_FACTOR, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \ + M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, S3::DEFAULT_UPLOAD_PART_SIZE_MULTIPLY_PARTS_COUNT_THRESHOLD, "Each time this number of parts was uploaded to S3, s3_min_upload_part_size is multiplied by s3_upload_part_size_multiply_factor.", 0) \ + M(UInt64, s3_max_part_number, S3::DEFAULT_MAX_PART_NUMBER, "Maximum part number number for s3 upload part.", 0) \ + M(UInt64, s3_max_single_operation_copy_size, S3::DEFAULT_MAX_SINGLE_OPERATION_COPY_SIZE, "Maximum size for a single copy operation in s3", 0) \ + M(UInt64, azure_upload_part_size_multiply_factor, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage.", 0) \ + M(UInt64, azure_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor.", 0) \ + M(UInt64, s3_max_inflight_parts_for_one_file, S3::DEFAULT_MAX_INFLIGHT_PARTS_FOR_ONE_FILE, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ + M(UInt64, azure_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ + M(UInt64, s3_max_single_part_upload_size, S3::DEFAULT_MAX_SINGLE_PART_UPLOAD_SIZE, "The maximum size of object to upload using singlepart upload to S3.", 0) \ + M(UInt64, azure_max_single_part_upload_size, 100*1024*1024, "The maximum size of object to upload using singlepart upload to Azure blob storage.", 0) \ + M(UInt64, azure_max_single_part_copy_size, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage.", 0) \ + M(UInt64, s3_max_single_read_retries, S3::DEFAULT_MAX_SINGLE_READ_TRIES, "The maximum number of retries during single S3 read.", 0) \ + M(UInt64, azure_max_single_read_retries, 4, "The maximum number of retries during single Azure blob storage read.", 0) \ + M(UInt64, azure_max_unexpected_write_error_retries, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write", 0) \ + M(UInt64, s3_max_unexpected_write_error_retries, S3::DEFAULT_MAX_UNEXPECTED_WRITE_ERROR_RETRIES, "The maximum number of retries in case of unexpected errors during S3 write.", 0) \ + M(UInt64, s3_max_redirects, S3::DEFAULT_MAX_REDIRECTS, "Max number of S3 redirects hops allowed.", 0) \ + M(UInt64, s3_max_connections, S3::DEFAULT_MAX_CONNECTIONS, "The maximum number of connections per server.", 0) \ + M(UInt64, s3_max_get_rps, 0, "Limit on S3 GET request per second rate before throttling. Zero means unlimited.", 0) \ + M(UInt64, s3_max_get_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_get_rps`", 0) \ + M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ + M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ + M(UInt64, s3_list_object_keys_size, S3::DEFAULT_LIST_OBJECT_KEYS_SIZE, "Maximum number of files that could be returned in batch by ListObject request", 0) \ + M(Bool, s3_use_adaptive_timeouts, S3::DEFAULT_USE_ADAPTIVE_TIMEOUTS, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ + M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ + M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ + M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ + M(Bool, s3_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in s3 engine tables", 0) \ + M(Bool, s3_skip_empty_files, false, "Allow to skip empty files in s3 table engine", 0) \ + M(Bool, azure_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in azure engine tables", 0) \ + M(Bool, s3_check_objects_after_upload, false, "Check each uploaded object to s3 with head request to be sure that upload was successful", 0) \ + M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ + M(Bool, azure_allow_parallel_part_upload, true, "Use multiple threads for azure multipart upload.", 0) \ + M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ + M(Bool, hdfs_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ + M(Bool, azure_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ + M(Bool, s3_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in S3 table engine", 0) \ + M(Bool, hdfs_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in HDFS table engine", 0) \ + M(Bool, azure_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in AzureBlobStorage table engine", 0) \ + M(UInt64, azure_sdk_max_retries, 10, "Maximum number of retries in azure sdk", 0) \ + M(UInt64, azure_sdk_retry_initial_backoff_ms, 10, "Minimal backoff between retries in azure sdk", 0) \ + M(UInt64, azure_sdk_retry_max_backoff_ms, 1000, "Maximal backoff between retries in azure sdk", 0) \ + M(Bool, s3_validate_request_settings, true, "Validate S3 request settings", 0) \ + M(Bool, s3_disable_checksum, S3::DEFAULT_DISABLE_CHECKSUM, "Do not calculate a checksum when sending a file to S3. This speeds up writes by avoiding excessive processing passes on a file. It is mostly safe as the data of MergeTree tables is checksummed by ClickHouse anyway, and when S3 is accessed with HTTPS, the TLS layer already provides integrity while transferring through the network. While additional checksums on S3 give defense in depth.", 0) \ + M(UInt64, s3_retry_attempts, S3::DEFAULT_RETRY_ATTEMPTS, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ + M(UInt64, s3_request_timeout_ms, S3::DEFAULT_REQUEST_TIMEOUT_MS, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ + M(UInt64, s3_connect_timeout_ms, S3::DEFAULT_CONNECT_TIMEOUT_MS, "Connection timeout for host from s3 disks.", 0) \ + M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ + M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ + M(Bool, s3queue_enable_logging_to_s3queue_log, false, "Enable writing to system.s3queue_log. The value can be overwritten per table with table settings", 0) \ + M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \ + M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \ + M(Bool, hdfs_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in hdfs engine tables", 0) \ + M(Bool, hdfs_skip_empty_files, false, "Allow to skip empty files in hdfs table engine", 0) \ + M(Bool, azure_skip_empty_files, false, "Allow to skip empty files in azure table engine", 0) \ + M(UInt64, hsts_max_age, 0, "Expired time for hsts. 0 means disable HSTS.", 0) \ + M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ + M(Bool, use_uncompressed_cache, false, "Whether to use the cache of uncompressed blocks.", 0) \ + M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ + M(UInt64, max_remote_read_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for read.", 0) \ + M(UInt64, max_remote_write_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for write.", 0) \ + M(UInt64, max_local_read_bandwidth, 0, "The maximum speed of local reads in bytes per second.", 0) \ + M(UInt64, max_local_write_bandwidth, 0, "The maximum speed of local writes in bytes per second.", 0) \ + M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams, and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \ + M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \ + M(Bool, dictionary_validate_primary_key_type, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64.", 0) \ + M(Bool, distributed_insert_skip_read_only_replicas, false, "If true, INSERT into Distributed will skip read-only replicas.", 0) \ + M(Bool, distributed_foreground_insert, false, "If setting is enabled, insert query into distributed waits until data are sent to all nodes in a cluster. \n\nEnables or disables synchronous data insertion into a `Distributed` table.\n\nBy default, when inserting data into a Distributed table, the ClickHouse server sends data to cluster nodes in the background. When `distributed_foreground_insert` = 1, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true).", 0) ALIAS(insert_distributed_sync) \ + M(UInt64, distributed_background_insert_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) ALIAS(insert_distributed_timeout) \ + M(Milliseconds, distributed_background_insert_sleep_time_ms, 100, "Sleep time for background INSERTs into Distributed, in case of any errors delay grows exponentially.", 0) ALIAS(distributed_directory_monitor_sleep_time_ms) \ + M(Milliseconds, distributed_background_insert_max_sleep_time_ms, 30000, "Maximum sleep time for background INSERTs into Distributed, it limits exponential growth too.", 0) ALIAS(distributed_directory_monitor_max_sleep_time_ms) \ + \ + M(Bool, distributed_background_insert_batch, false, "Should background INSERTs into Distributed be batched into bigger blocks.", 0) ALIAS(distributed_directory_monitor_batch_inserts) \ + M(Bool, distributed_background_insert_split_batch_on_failure, false, "Should batches of the background INSERT into Distributed be split into smaller batches in case of failures.", 0) ALIAS(distributed_directory_monitor_split_batch_on_failure) \ + \ + M(Bool, optimize_move_to_prewhere, true, "Allows disabling WHERE to PREWHERE optimization in SELECT queries from MergeTree.", 0) \ + M(Bool, optimize_move_to_prewhere_if_final, false, "If query has `FINAL`, the optimization `move_to_prewhere` is not always correct and it is enabled only if both settings `optimize_move_to_prewhere` and `optimize_move_to_prewhere_if_final` are turned on", 0) \ + M(Bool, move_all_conditions_to_prewhere, true, "Move all viable conditions from WHERE to PREWHERE", 0) \ + M(Bool, enable_multiple_prewhere_read_steps, true, "Move more conditions from WHERE to PREWHERE and do reads from disk and filtering in multiple steps if there are multiple conditions combined with AND", 0) \ + M(Bool, move_primary_key_columns_to_end_of_prewhere, true, "Move PREWHERE conditions containing primary key columns to the end of AND chain. It is likely that these conditions are taken into account during primary key analysis and thus will not contribute a lot to PREWHERE filtering.", 0) \ + \ + M(UInt64, alter_sync, 1, "Wait for actions to manipulate the partitions. 0 - do not wait, 1 - wait for execution only of itself, 2 - wait for everyone.", 0) ALIAS(replication_alter_partitions_sync) \ + M(Int64, replication_wait_for_inactive_replica_timeout, 120, "Wait for inactive replica to execute ALTER/OPTIMIZE. Time in seconds, 0 - do not wait, negative - wait for unlimited time.", 0) \ + M(Bool, alter_move_to_space_execute_async, false, "Execute ALTER TABLE MOVE ... TO [DISK|VOLUME] asynchronously", 0) \ + \ + M(LoadBalancing, load_balancing, LoadBalancing::RANDOM, "Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing.", 0) \ + M(UInt64, load_balancing_first_offset, 0, "Which replica to preferably send a query when FIRST_OR_RANDOM load balancing strategy is used.", 0) \ + \ + M(TotalsMode, totals_mode, TotalsMode::AFTER_HAVING_EXCLUSIVE, "How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = ‘any’ are present.", IMPORTANT) \ + M(Float, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.", 0) \ + \ + M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \ + M(Bool, allow_suspicious_fixed_string_types, false, "In CREATE TABLE statement allows creating columns of type FixedString(n) with n > 256. FixedString with length >= 256 is suspicious and most likely indicates misuse", 0) \ + M(Bool, allow_suspicious_indices, false, "Reject primary/secondary indexes and sorting keys with identical expressions", 0) \ + M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \ + M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \ + M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \ + M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code. Due to a bug in the LLVM compiler infrastructure, on AArch64 machines, it is known to lead to a nullptr dereference and, consequently, server crash. Do not enable this setting.", 0) \ + M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ + M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ + M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ + M(Bool, compile_sort_description, true, "Compile sort description to native code.", 0) \ + M(UInt64, min_count_to_compile_sort_description, 3, "The number of identical sort descriptions before they are JIT-compiled", 0) \ + M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ + M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ + M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ + M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ + M(Bool, enable_memory_bound_merging_of_aggregation_results, true, "Enable memory bound merging strategy for aggregation.", 0) \ + M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ + M(Bool, enable_extended_results_for_datetime_functions, false, "Enable date functions like toLastDayOfMonth return Date32 results (instead of Date results) for Date32/DateTime64 arguments.", 0) \ + M(Bool, allow_nonconst_timezone_arguments, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()", 0) \ + M(Bool, function_locate_has_mysql_compatible_argument_order, true, "Function locate() has arguments (needle, haystack[, start_pos]) like in MySQL instead of (haystack, needle[, start_pos]) like function position()", 0) \ + \ + M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ + \ + M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \ + \ + M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards. Shard is marked as unavailable when: 1) The shard cannot be reached due to a connection failure. 2) Shard is unresolvable through DNS. 3) Table does not exist on the shard.", 0) \ + \ + M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard; if set to 1 - SELECT is executed on each shard; if set to 2 - SELECT and INSERT are executed on each shard", 0) \ + M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ + M(UInt64, distributed_push_down_limit, 1, "If 1, LIMIT will be applied on each shard separately. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.", 0) \ + M(Bool, optimize_distributed_group_by_sharding_key, true, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ + M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ + M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ + M(Bool, optimize_skip_unused_shards_rewrite_in, true, "Rewrite IN in query for remote shards to exclude values that does not belong to the shard (requires optimize_skip_unused_shards)", 0) \ + M(Bool, allow_nondeterministic_optimize_skip_unused_shards, false, "Allow non-deterministic functions (includes dictGet) in sharding_key for optimize_skip_unused_shards", 0) \ + M(UInt64, force_optimize_skip_unused_shards, 0, "Throw an exception if unused shards cannot be skipped (1 - throw only if the table has the sharding key, 2 - always throw.", 0) \ + M(UInt64, optimize_skip_unused_shards_nesting, 0, "Same as optimize_skip_unused_shards, but accept nesting level until which it will work.", 0) \ + M(UInt64, force_optimize_skip_unused_shards_nesting, 0, "Same as force_optimize_skip_unused_shards, but accept nesting level until which it will work.", 0) \ + \ + M(Bool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ + M(UInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ + M(Bool, output_format_parallel_formatting, true, "Enable parallel formatting for some data formats.", 0) \ + M(UInt64, output_format_compression_level, 3, "Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when inserting to table function `file`, `url`, `hdfs`, `s3`, and `azureBlobStorage`.", 0) \ + M(UInt64, output_format_compression_zstd_window_log, 0, "Can be used when the output compression method is `zstd`. If greater than `0`, this setting explicitly sets compression window size (power of `2`) and enables a long-range mode for zstd compression.", 0) \ + \ + M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ + M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ + M(UInt64, merge_tree_min_rows_for_seek, 0, "You can skip reading more than that number of rows at the price of one seek per file.", 0) \ + M(UInt64, merge_tree_min_bytes_for_seek, 0, "You can skip reading more than that number of bytes at the price of one seek per file.", 0) \ + M(UInt64, merge_tree_coarse_index_granularity, 8, "If the index segment can contain the required keys, divide it into as many parts and recursively check them.", 0) \ + M(UInt64, merge_tree_max_rows_to_use_cache, (128 * 8192), "The maximum number of rows per request, to use the cache of uncompressed data. If the request is large, the cache is not used. (For large queries not to flush out the cache.)", 0) \ + M(UInt64, merge_tree_max_bytes_to_use_cache, (192 * 10 * 1024 * 1024), "The maximum number of bytes per request, to use the cache of uncompressed data. If the request is large, the cache is not used. (For large queries not to flush out the cache.)", 0) \ + M(Bool, do_not_merge_across_partitions_select_final, false, "Merge parts only in one partition in select final", 0) \ + M(Bool, split_parts_ranges_into_intersecting_and_non_intersecting_final, true, "Split parts ranges into intersecting and non intersecting during FINAL optimization", 0) \ + M(Bool, split_intersecting_parts_ranges_into_layers_final, true, "Split intersecting parts ranges into layers during FINAL optimization", 0) \ + \ + M(UInt64, mysql_max_rows_to_insert, 65536, "The maximum number of rows in MySQL batch insertion of the MySQL storage engine", 0) \ + M(Bool, mysql_map_string_to_text_in_show_columns, true, "If enabled, String type will be mapped to TEXT in SHOW [FULL] COLUMNS, BLOB otherwise. Has an effect only when the connection is made through the MySQL wire protocol.", 0) \ + M(Bool, mysql_map_fixed_string_to_text_in_show_columns, true, "If enabled, FixedString type will be mapped to TEXT in SHOW [FULL] COLUMNS, BLOB otherwise. Has an effect only when the connection is made through the MySQL wire protocol.", 0) \ + \ + M(UInt64, optimize_min_equality_disjunction_chain_length, 3, "The minimum length of the expression `expr = x1 OR ... expr = xN` for optimization ", 0) \ + M(UInt64, optimize_min_inequality_conjunction_chain_length, 3, "The minimum length of the expression `expr <> x1 AND ... expr <> xN` for optimization ", 0) \ + \ + M(UInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for reading the data with O_DIRECT option during SELECT queries execution. 0 - disabled.", 0) \ + M(UInt64, min_bytes_to_use_mmap_io, 0, "The minimum number of bytes for reading the data with mmap option during SELECT queries execution. 0 - disabled.", 0) \ + M(Bool, checksum_on_read, true, "Validate checksums on reading. It is enabled by default and should be always enabled in production. Please do not expect any benefits in disabling this setting. It may only be used for experiments and benchmarks. The setting only applicable for tables of MergeTree family. Checksums are always validated for other table engines and when receiving data over network.", 0) \ + \ + M(Bool, force_index_by_date, false, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \ + M(Bool, force_primary_key, false, "Throw an exception if there is primary key in a table, and it is not used.", 0) \ + M(Bool, use_skip_indexes, true, "Use data skipping indexes during query execution.", 0) \ + M(Bool, use_skip_indexes_if_final, false, "If query has FINAL, then skipping data based on indexes may produce incorrect result, hence disabled by default.", 0) \ + M(Bool, materialize_skip_indexes_on_insert, true, "If true skip indexes are calculated on inserts, otherwise skip indexes will be calculated only during merges", 0) \ + M(Bool, materialize_statistics_on_insert, true, "If true statistics are calculated on inserts, otherwise statistics will be calculated only during merges", 0) \ + M(String, ignore_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be excluded during query execution.", 0) \ + \ + M(String, force_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be used during query execution, otherwise an exception will be thrown.", 0) \ + \ + M(Float, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \ + M(Float, max_streams_multiplier_for_merge_tables, 5, "Ask more streams when reading from Merge table. Streams will be spread across tables that Merge table will use. This allows more even distribution of work across threads and especially helpful when merged tables differ in size.", 0) \ + \ + M(String, network_compression_method, "LZ4", "Allows you to select the method of data compression when writing.", 0) \ + \ + M(Int64, network_zstd_compression_level, 1, "Allows you to select the level of ZSTD compression.", 0) \ + \ + M(Int64, zstd_window_log_max, 0, "Allows you to select the max window log of ZSTD (it will not be used for MergeTree family)", 0) \ + \ + M(UInt64, priority, 0, "Priority of the query. 1 - the highest, higher value - lower priority; 0 - do not use priorities.", 0) \ + M(Int64, os_thread_priority, 0, "If non zero - set corresponding 'nice' value for query processing threads. Can be used to adjust query priority for OS scheduler.", 0) \ + \ + M(Bool, log_queries, true, "Log requests and write the log to the system table.", 0) \ + M(Bool, log_formatted_queries, false, "Log formatted queries and write the log to the system table.", 0) \ + M(LogQueriesType, log_queries_min_type, QueryLogElementType::QUERY_START, "Minimal type in query_log to log, possible values (from low to high): QUERY_START, QUERY_FINISH, EXCEPTION_BEFORE_START, EXCEPTION_WHILE_PROCESSING.", 0) \ + M(Milliseconds, log_queries_min_query_duration_ms, 0, "Minimal time for the query to run, to get to the query_log/query_thread_log/query_views_log.", 0) \ + M(UInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \ + M(Float, log_queries_probability, 1., "Log queries with the specified probability.", 0) \ + \ + M(Bool, log_processors_profiles, true, "Log Processors profile events.", 0) \ + M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \ + \ + M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \ + M(UInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.", 0) \ + \ + M(Bool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of inserting blocks should be performed", 0) \ + M(Bool, async_insert_deduplicate, false, "For async INSERT queries in the replicated table, specifies that deduplication of inserting blocks should be performed", 0) \ + \ + M(UInt64Auto, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled, 'auto' - use majority", 0) \ + M(Milliseconds, insert_quorum_timeout, 600000, "If the quorum of replicas did not meet in specified time (in milliseconds), exception will be thrown and insertion is aborted.", 0) \ + M(Bool, insert_quorum_parallel, true, "For quorum INSERT queries - enable to make parallel inserts without linearizability", 0) \ + M(UInt64, select_sequential_consistency, 0, "For SELECT queries from the replicated table, throw an exception if the replica does not have a chunk written with the quorum; do not read the parts that have not yet been written with the quorum.", 0) \ + M(UInt64, table_function_remote_max_addresses, 1000, "The maximum number of different shards and the maximum number of replicas of one shard in the `remote` function.", 0) \ + M(Milliseconds, read_backoff_min_latency_ms, 1000, "Setting to reduce the number of threads in case of slow reads. Pay attention only to reads that took at least that much time.", 0) \ + M(UInt64, read_backoff_max_throughput, 1048576, "Settings to reduce the number of threads in case of slow reads. Count events when the read bandwidth is less than that many bytes per second.", 0) \ + M(Milliseconds, read_backoff_min_interval_between_events_ms, 1000, "Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time.", 0) \ + M(UInt64, read_backoff_min_events, 2, "Settings to reduce the number of threads in case of slow reads. The number of events after which the number of threads will be reduced.", 0) \ + \ + M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ + \ + M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ + M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ + \ + M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \ + M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \ + \ + M(Bool, http_native_compression_disable_checksumming_on_decompress, false, "If you uncompress the POST data from the client compressed by the native format, do not check the checksum.", 0) \ + \ + M(String, count_distinct_implementation, "uniqExact", "What aggregate function to use for implementation of count(DISTINCT ...)", 0) \ + \ + M(Bool, add_http_cors_header, false, "Write add http CORS header.", 0) \ + \ + M(UInt64, max_http_get_redirects, 0, "Max number of http GET redirects hops allowed. Ensures additional security measures are in place to prevent a malicious server to redirect your requests to unexpected services.\n\nIt is the case when an external server redirects to another address, but that address appears to be internal to the company's infrastructure, and by sending an HTTP request to an internal server, you could request an internal API from the internal network, bypassing the auth, or even query other services, such as Redis or Memcached. When you don't have an internal infrastructure (including something running on your localhost), or you trust the server, it is safe to allow redirects. Although keep in mind, that if the URL uses HTTP instead of HTTPS, and you will have to trust not only the remote server but also your ISP and every network in the middle.", 0) \ + \ + M(Bool, use_client_time_zone, false, "Use client timezone for interpreting DateTime string values, instead of adopting server timezone.", 0) \ + \ + M(Bool, send_progress_in_http_headers, false, "Send progress notifications using X-ClickHouse-Progress headers. Some clients do not support high amount of HTTP headers (Python requests in particular), so it is disabled by default.", 0) \ + \ + M(UInt64, http_headers_progress_interval_ms, 100, "Do not send HTTP headers X-ClickHouse-Progress more frequently than at each specified interval.", 0) \ + M(Bool, http_wait_end_of_query, false, "Enable HTTP response buffering on the server-side.", 0) \ + M(Bool, http_write_exception_in_output_format, true, "Write exception in output format to produce valid output. Works with JSON and XML formats.", 0) \ + M(UInt64, http_response_buffer_size, 0, "The number of bytes to buffer in the server memory before sending a HTTP response to the client or flushing to disk (when http_wait_end_of_query is enabled).", 0) \ + \ + M(Bool, fsync_metadata, true, "Do fsync after changing metadata for tables and databases (.sql files). Could be disabled in case of poor latency on server with high load of DDL queries and high load of disk subsystem.", 0) \ + \ + M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ + \ + M(Int32, join_output_by_rowlist_perkey_rows_threshold, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join.", 0) \ + M(JoinStrictness, join_default_strictness, JoinStrictness::All, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ + M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ + M(Bool, single_join_prefer_left_table, true, "For single JOIN in case of identifier ambiguity prefer left table", IMPORTANT) \ + \ + M(UInt64, preferred_block_size_bytes, 1000000, "This setting adjusts the data block size for query processing and represents additional fine tune to the more rough 'max_block_size' setting. If the columns are large and with 'max_block_size' rows the block size is likely to be larger than the specified amount of bytes, its size will be lowered for better CPU cache locality.", 0) \ + \ + M(UInt64, max_replica_delay_for_distributed_queries, 300, "If set, distributed queries of Replicated tables will choose servers with replication delay in seconds less than the specified value (not inclusive). Zero means do not take delay into account.", 0) \ + M(Bool, fallback_to_stale_replicas_for_distributed_queries, true, "Suppose max_replica_delay_for_distributed_queries is set and all replicas for the queried table are stale. If this setting is enabled, the query will be performed anyway, otherwise the error will be reported.", 0) \ + M(UInt64, preferred_max_column_in_block_size_bytes, 0, "Limit on max column size in block while reading. Helps to decrease cache misses count. Should be close to L2 cache size.", 0) \ + \ + M(UInt64, parts_to_delay_insert, 0, "If the destination table contains at least that many active parts in a single partition, artificially slow down insert into table.", 0) \ + M(UInt64, parts_to_throw_insert, 0, "If more than this number active parts in a single partition of the destination table, throw 'Too many parts ...' exception.", 0) \ + M(UInt64, number_of_mutations_to_delay, 0, "If the mutated table contains at least that many unfinished mutations, artificially slow down mutations of table. 0 - disabled", 0) \ + M(UInt64, number_of_mutations_to_throw, 0, "If the mutated table contains at least that many unfinished mutations, throw 'Too many mutations ...' exception. 0 - disabled", 0) \ + M(Int64, distributed_ddl_task_timeout, 180, "Timeout for DDL query responses from all hosts in cluster. If a ddl request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. Negative value means infinite. Zero means async mode.", 0) \ + M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ + M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ + \ + M(Bool, final, false, "Query with the FINAL modifier by default. If the engine does not support final, it does not have any effect. On queries with multiple tables final is applied only on those that support it. It also works on distributed tables", 0) \ + \ + M(Bool, partial_result_on_first_cancel, false, "Allows query to return a partial result after cancel.", 0) \ + \ + M(Bool, ignore_on_cluster_for_replicated_udf_queries, false, "Ignore ON CLUSTER clause for replicated UDF management queries.", 0) \ + M(Bool, ignore_on_cluster_for_replicated_access_entities_queries, false, "Ignore ON CLUSTER clause for replicated access entities management queries.", 0) \ + M(Bool, ignore_on_cluster_for_replicated_named_collections_queries, false, "Ignore ON CLUSTER clause for replicated named collections management queries.", 0) \ + /** Settings for testing hedged requests */ \ + M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ + M(Milliseconds, sleep_in_send_data_ms, 0, "Time to sleep in sending data in TCPHandler", 0) \ + M(Milliseconds, sleep_after_receiving_query_ms, 0, "Time to sleep after receiving query in TCPHandler", 0) \ + M(UInt64, unknown_packet_in_send_data, 0, "Send unknown packet instead of data Nth data packet", 0) \ + \ + M(Bool, insert_allow_materialized_columns, false, "If setting is enabled, Allow materialized columns in INSERT.", 0) \ + M(Seconds, http_connection_timeout, DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT, "HTTP connection timeout.", 0) \ + M(Seconds, http_send_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP send timeout", 0) \ + M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \ + M(UInt64, http_max_uri_size, 1048576, "Maximum URI length of HTTP request", 0) \ + M(UInt64, http_max_fields, 1000000, "Maximum number of fields in HTTP header", 0) \ + M(UInt64, http_max_field_name_size, 128 * 1024, "Maximum length of field name in HTTP header", 0) \ + M(UInt64, http_max_field_value_size, 128 * 1024, "Maximum length of field value in HTTP header", 0) \ + M(Bool, http_skip_not_found_url_for_globs, true, "Skip URLs for globs with HTTP_NOT_FOUND error", 0) \ + M(Bool, http_make_head_request, true, "Allows the execution of a `HEAD` request while reading data from HTTP to retrieve information about the file to be read, such as its size", 0) \ + M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \ + M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \ + M(UInt64, use_index_for_in_with_subqueries_max_values, 0, "The maximum size of set in the right hand side of the IN operator to use table index for filtering. It allows to avoid performance degradation and higher memory usage due to preparation of additional data structures for large queries. Zero means no limit.", 0) \ + M(Bool, analyze_index_with_space_filling_curves, true, "If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.", 0) \ + M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \ + M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ + M(Bool, empty_result_for_aggregation_by_constant_keys_on_empty_set, true, "Return empty result when aggregating by constant keys on empty set.", 0) \ + M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ + M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ + M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \ + M(Bool, enable_zstd_qat_codec, false, "Enable/disable the ZSTD_QAT codec.", 0) \ + M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ + M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ + M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ + M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \ + M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ + M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ + M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ + \ + M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ + M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \ + \ + \ + /** Limits during query execution are part of the settings. \ + * Used to provide a more safe execution of queries from the user interface. \ + * Basically, limits are checked for each block (not every row). That is, the limits can be slightly violated. \ + * Almost all limits apply only to SELECTs. \ + * Almost all limits apply to each stream individually. \ + */ \ + \ + M(UInt64, max_rows_to_read, 0, "Limit on read rows from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \ + M(UInt64, max_bytes_to_read, 0, "Limit on read bytes (after decompression) from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \ + M(OverflowMode, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + M(UInt64, max_rows_to_read_leaf, 0, "Limit on read rows on the leaf nodes for distributed queries. Limit is applied for local reads only, excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \ + M(UInt64, max_bytes_to_read_leaf, 0, "Limit on read bytes (after decompression) on the leaf nodes for distributed queries. Limit is applied for local reads only, excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \ + M(OverflowMode, read_overflow_mode_leaf, OverflowMode::THROW, "What to do when the leaf limit is exceeded.", 0) \ + \ + M(UInt64, max_rows_to_group_by, 0, "If aggregation during GROUP BY is generating more than the specified number of rows (unique GROUP BY keys), the behavior will be determined by the 'group_by_overflow_mode' which by default is - throw an exception, but can be also switched to an approximate GROUP BY mode.", 0) \ + M(OverflowModeGroupBy, group_by_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + M(UInt64, max_bytes_before_external_group_by, 0, "If memory usage during GROUP BY operation is exceeding this threshold in bytes, activate the 'external aggregation' mode (spill data to disk). Recommended value is half of available system memory.", 0) \ + \ + M(UInt64, max_rows_to_sort, 0, "If more than the specified amount of records have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \ + M(UInt64, max_bytes_to_sort, 0, "If more than the specified amount of (uncompressed) bytes have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \ + M(OverflowMode, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + M(UInt64, prefer_external_sort_block_bytes, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging.", 0) \ + M(UInt64, max_bytes_before_external_sort, 0, "If memory usage during ORDER BY operation is exceeding this threshold in bytes, activate the 'external sorting' mode (spill data to disk). Recommended value is half of available system memory.", 0) \ + M(UInt64, max_bytes_before_remerge_sort, 1000000000, "In case of ORDER BY with LIMIT, when memory usage is higher than specified threshold, perform additional steps of merging blocks before final merge to keep just top LIMIT rows.", 0) \ + M(Float, remerge_sort_lowered_memory_bytes_ratio, 2., "If memory usage after remerge does not reduced by this ratio, remerge will be disabled.", 0) \ + \ + M(UInt64, max_result_rows, 0, "Limit on result size in rows. The query will stop after processing a block of data if the threshold is met, but it will not cut the last block of the result, therefore the result size can be larger than the threshold.", 0) \ + M(UInt64, max_result_bytes, 0, "Limit on result size in bytes (uncompressed). The query will stop after processing a block of data if the threshold is met, but it will not cut the last block of the result, therefore the result size can be larger than the threshold. Caveats: the result size in memory is taken into account for this threshold. Even if the result size is small, it can reference larger data structures in memory, representing dictionaries of LowCardinality columns, and Arenas of AggregateFunction columns, so the threshold can be exceeded despite the small result size. The setting is fairly low level and should be used with caution.", 0) \ + M(OverflowMode, result_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + /* TODO: Check also when merging and finalizing aggregate functions. */ \ + M(Seconds, max_execution_time, 0, "If query runtime exceeds the specified number of seconds, the behavior will be determined by the 'timeout_overflow_mode', which by default is - throw an exception. Note that the timeout is checked and query can stop only in designated places during data processing. It currently cannot stop during merging of aggregation states or during query analysis, and the actual run time will be higher than the value of this setting.", 0) \ + M(OverflowMode, timeout_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + M(Seconds, max_execution_time_leaf, 0, "Similar semantic to max_execution_time but only apply on leaf node for distributed queries, the time out behavior will be determined by 'timeout_overflow_mode_leaf' which by default is - throw an exception", 0) \ + M(OverflowMode, timeout_overflow_mode_leaf, OverflowMode::THROW, "What to do when the leaf limit is exceeded.", 0) \ + \ + M(UInt64, min_execution_speed, 0, "Minimum number of execution rows per second.", 0) \ + M(UInt64, max_execution_speed, 0, "Maximum number of execution rows per second.", 0) \ + M(UInt64, min_execution_speed_bytes, 0, "Minimum number of execution bytes per second.", 0) \ + M(UInt64, max_execution_speed_bytes, 0, "Maximum number of execution bytes per second.", 0) \ + M(Seconds, timeout_before_checking_execution_speed, 10, "Check that the speed is not too low after the specified time has elapsed.", 0) \ + M(Seconds, max_estimated_execution_time, 0, "Maximum query estimate execution time in seconds.", 0) \ + \ + M(UInt64, max_columns_to_read, 0, "If a query requires reading more than specified number of columns, exception is thrown. Zero value means unlimited. This setting is useful to prevent too complex queries.", 0) \ + M(UInt64, max_temporary_columns, 0, "If a query generates more than the specified number of temporary columns in memory as a result of intermediate calculation, exception is thrown. Zero value means unlimited. This setting is useful to prevent too complex queries.", 0) \ + M(UInt64, max_temporary_non_const_columns, 0, "Similar to the 'max_temporary_columns' setting but applies only to non-constant columns. This makes sense, because constant columns are cheap and it is reasonable to allow more of them.", 0) \ + \ + M(UInt64, max_sessions_for_user, 0, "Maximum number of simultaneous sessions for a user.", 0) \ + \ + M(UInt64, max_subquery_depth, 100, "If a query has more than the specified number of nested subqueries, throw an exception. This allows you to have a sanity check to protect the users of your cluster from going insane with their queries.", 0) \ + M(UInt64, max_analyze_depth, 5000, "Maximum number of analyses performed by interpreter.", 0) \ + M(UInt64, max_ast_depth, 1000, "Maximum depth of query syntax tree. Checked after parsing.", 0) \ + M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \ + M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \ + \ + M(UInt64, readonly, 0, "0 - no read-only restrictions. 1 - only read requests, as well as changing explicitly allowed settings. 2 - only read requests, as well as changing settings, except for the 'readonly' setting.", 0) \ + \ + M(UInt64, max_rows_in_set, 0, "Maximum size of the set (in number of elements) resulting from the execution of the IN section.", 0) \ + M(UInt64, max_bytes_in_set, 0, "Maximum size of the set (in bytes in memory) resulting from the execution of the IN section.", 0) \ + M(OverflowMode, set_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + M(UInt64, max_rows_in_join, 0, "Maximum size of the hash table for JOIN (in number of rows).", 0) \ + M(UInt64, max_bytes_in_join, 0, "Maximum size of the hash table for JOIN (in number of bytes in memory).", 0) \ + M(OverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + M(Bool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key. Can be applied only to hash join and storage join.", IMPORTANT) \ + M(JoinAlgorithm, join_algorithm, JoinAlgorithm::DEFAULT, "Specify join algorithm.", 0) \ + M(UInt64, cross_join_min_rows_to_compress, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.", 0) \ + M(UInt64, cross_join_min_bytes_to_compress, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.", 0) \ + M(UInt64, default_max_bytes_in_join, 1000000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \ + M(UInt64, partial_merge_join_left_table_buffer_bytes, 0, "If not 0 group left table blocks in bigger ones for left-side table in partial merge join. It uses up to 2x of specified memory per joining thread.", 0) \ + M(UInt64, partial_merge_join_rows_in_right_blocks, 65536, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ + M(UInt64, join_on_disk_max_files_to_merge, 64, "For MergeJoin on disk set how much files it's allowed to sort simultaneously. Then this value bigger then more memory used and then less disk I/O needed. Minimum is 2.", 0) \ + M(UInt64, max_rows_in_set_to_optimize_join, 0, "Maximal size of the set to filter joined tables by each other row sets before joining. 0 - disable.", 0) \ + \ + M(Bool, compatibility_ignore_collation_in_create_table, true, "Compatibility ignore collation in create table", 0) \ + \ + M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files produced by (JOINs, external GROUP BY, external ORDER BY). I.e. LZ4, NONE.", 0) \ + \ + M(UInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ + M(UInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ + M(OverflowMode, transfer_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + M(UInt64, max_rows_in_distinct, 0, "Maximum number of elements during execution of DISTINCT.", 0) \ + M(UInt64, max_bytes_in_distinct, 0, "Maximum total size of state (in uncompressed bytes) in memory for the execution of DISTINCT.", 0) \ + M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ + \ + M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ + M(UInt64, memory_overcommit_ratio_denominator, 1_GiB, "It represents soft memory limit on the user level. This value is used to compute query overcommit ratio.", 0) \ + M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ + M(UInt64, memory_overcommit_ratio_denominator_for_user, 1_GiB, "It represents soft memory limit on the global level. This value is used to compute query overcommit ratio.", 0) \ + M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \ + M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ + M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation (can be changed with `memory_profiler_sample_min_allocation_size` and `memory_profiler_sample_max_allocation_size`). Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ + M(UInt64, memory_profiler_sample_min_allocation_size, 0, "Collect random allocations of size greater or equal than specified value with probability equal to `memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \ + M(UInt64, memory_profiler_sample_max_allocation_size, 0, "Collect random allocations of size less or equal than specified value with probability equal to `memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \ + M(Bool, trace_profile_events, false, "Send to system.trace_log profile event and value of increment on each increment with 'ProfileEvent' trace_type", 0) \ + \ + M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown.", 0) \ + \ + M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ + M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ + M(UInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.", 0)\ + M(UInt64, max_network_bandwidth_for_all_users, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running queries. Zero means unlimited.", 0) \ + \ + M(UInt64, max_temporary_data_on_disk_size_for_user, 0, "The maximum amount of data consumed by temporary files on disk in bytes for all concurrently running user queries. Zero means unlimited.", 0)\ + M(UInt64, max_temporary_data_on_disk_size_for_query, 0, "The maximum amount of data consumed by temporary files on disk in bytes for all concurrently running queries. Zero means unlimited.", 0)\ + \ + M(UInt64, backup_restore_keeper_max_retries, 20, "Max retries for keeper operations during backup or restore", 0) \ + M(UInt64, backup_restore_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for [Zoo]Keeper operations during backup or restore", 0) \ + M(UInt64, backup_restore_keeper_retry_max_backoff_ms, 5000, "Max backoff timeout for [Zoo]Keeper operations during backup or restore", 0) \ + M(Float, backup_restore_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during backup or restore. Valid value is in interval [0.0f, 1.0f]", 0) \ + M(UInt64, backup_restore_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ + M(UInt64, backup_restore_keeper_value_max_size, 1048576, "Maximum size of data of a [Zoo]Keeper's node during backup", 0) \ + M(UInt64, backup_restore_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup or restore", 0) \ + M(UInt64, backup_restore_batch_size_for_keeper_multi, 1000, "Maximum size of batch for multi request to [Zoo]Keeper during backup or restore", 0) \ + M(UInt64, backup_restore_s3_retry_attempts, 1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore.", 0) \ + M(UInt64, max_backup_bandwidth, 0, "The maximum read speed in bytes per second for particular backup on server. Zero means unlimited.", 0) \ + \ + M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ + M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \ + M(Bool, log_query_threads, false, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \ + M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \ + M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \ + M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \ + M(String, send_logs_source_regexp, "", "Send server text logs with specified regexp to match log source name. Empty means all sources.", 0) \ + M(Bool, enable_optimize_predicate_expression, true, "If it is set to true, optimize predicates to subqueries.", 0) \ + M(Bool, enable_optimize_predicate_expression_to_final_subquery, true, "Allow push predicate to final subquery.", 0) \ + M(Bool, allow_push_predicate_when_subquery_contains_with, true, "Allows push predicate when subquery contains WITH clause", 0) \ + \ + M(UInt64, low_cardinality_max_dictionary_size, 8192, "Maximum size (in rows) of shared global dictionary for LowCardinality type.", 0) \ + M(Bool, low_cardinality_use_single_dictionary_for_part, false, "LowCardinality type serialization setting. If is true, than will use additional keys when global dictionary overflows. Otherwise, will create several shared dictionaries.", 0) \ + M(Bool, decimal_check_overflow, true, "Check overflow of decimal arithmetic/comparison operations", 0) \ + M(Bool, allow_custom_error_code_in_throwif, false, "Enable custom error code in function throwIf(). If true, thrown exceptions may have unexpected error codes.", 0) \ + \ + M(Bool, prefer_localhost_replica, true, "If it's true then queries will be always sent to local replica (if it exists). If it's false then replica to send a query will be chosen between local and remote ones according to load_balancing", 0) \ + M(UInt64, max_fetch_partition_retries_count, 5, "Amount of retries while fetching partition from another host.", 0) \ + M(UInt64, http_max_multipart_form_data_size, 1024 * 1024 * 1024, "Limit on size of multipart/form-data content. This setting cannot be parsed from URL parameters and should be set in user profile. Note that content is parsed and external tables are created in memory before start of query execution. And this is the only limit that has effect on that stage (limits on max memory usage and max execution time have no effect while reading HTTP form data).", 0) \ + M(Bool, calculate_text_stack_trace, true, "Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when huge amount of wrong queries are executed. In normal cases you should not disable this option.", 0) \ + M(Bool, enable_job_stack_trace, false, "Output stack trace of a job creator when job results in exception", 0) \ + M(Bool, allow_ddl, true, "If it is set to true, then a user is allowed to executed DDL queries.", 0) \ + M(Bool, parallel_view_processing, false, "Enables pushing to attached views concurrently instead of sequentially.", 0) \ + M(Bool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \ + M(Bool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \ + M(Bool, optimize_read_in_window_order, true, "Enable ORDER BY optimization in window clause for reading data in corresponding order in MergeTree tables.", 0) \ + M(Bool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \ + M(Bool, read_in_order_use_buffering, true, "Use buffering before merging while reading in order of primary key. It increases the parallelism of query execution", 0) \ + M(UInt64, aggregation_in_order_max_block_bytes, 50000000, "Maximal size of block in bytes accumulated during aggregation in order of primary key. Lower block size allows to parallelize more final merge stage of aggregation.", 0) \ + M(UInt64, read_in_order_two_level_merge_threshold, 100, "Minimal number of parts to read to run preliminary merge step during multithread reading in order of primary key.", 0) \ + M(Bool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \ + M(Bool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ + M(Bool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql', 'postgresql' and 'odbc' table functions.", 0) \ + M(Bool, external_table_strict_query, false, "If it is set to true, transforming expression to local filter is forbidden for queries to external tables.", 0) \ + \ + M(Bool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ + M(UInt64, max_hyperscan_regexp_length, 0, "Max length of regexp than can be used in hyperscan multi-match functions. Zero means unlimited.", 0) \ + M(UInt64, max_hyperscan_regexp_total_length, 0, "Max total length of all regexps than can be used in hyperscan multi-match functions (per every function). Zero means unlimited.", 0) \ + M(Bool, reject_expensive_hyperscan_regexps, true, "Reject patterns which will likely be expensive to evaluate with hyperscan (due to NFA state explosion)", 0) \ + M(Bool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ + M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ + M(Bool, splitby_max_substrings_includes_remaining_string, false, "Functions 'splitBy*()' with 'max_substrings' argument > 0 include the remaining string as last element in the result", 0) \ + \ + M(Bool, allow_execute_multiif_columnar, true, "Allow execute multiIf function columnar", 0) \ + M(Bool, formatdatetime_f_prints_single_zero, false, "Formatter '%f' in function 'formatDateTime()' prints a single zero instead of six zeros if the formatted value has no fractional seconds.", 0) \ + M(Bool, formatdatetime_parsedatetime_m_is_month_name, true, "Formatter '%M' in functions 'formatDateTime()' and 'parseDateTime()' print/parse the month name instead of minutes.", 0) \ + M(Bool, parsedatetime_parse_without_leading_zeros, true, "Formatters '%c', '%l' and '%k' in function 'parseDateTime()' parse months and hours without leading zeros.", 0) \ + M(Bool, formatdatetime_format_without_leading_zeros, false, "Formatters '%c', '%l' and '%k' in function 'formatDateTime()' print months and hours without leading zeros.", 0) \ + \ + M(UInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.", 0) \ + M(Bool, throw_on_max_partitions_per_insert_block, true, "Used with max_partitions_per_insert_block. If true (default), an exception will be thrown when max_partitions_per_insert_block is reached. If false, details of the insert query reaching this limit with the number of partitions will be logged. This can be useful if you're trying to understand the impact on users when changing max_partitions_per_insert_block.", 0) \ + M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ + M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ + M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ + M(UInt64, max_table_size_to_drop, 50000000000lu, "If size of a table is greater than this value (in bytes) than table could not be dropped with any DROP query.", 0) \ + M(UInt64, max_partition_size_to_drop, 50000000000lu, "Same as max_table_size_to_drop, but for the partitions.", 0) \ + \ + M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ + M(UInt64, postgresql_connection_attempt_timeout, 2, "Connection timeout to PostgreSQL table engine and database engine in seconds.", 0) \ + M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ + M(UInt64, postgresql_connection_pool_retries, 2, "Connection pool push/pop retries number for PostgreSQL table engine and database engine.", 0) \ + M(Bool, postgresql_connection_pool_auto_close_connection, false, "Close connection before returning connection to the pool.", 0) \ + M(UInt64, glob_expansion_max_elements, 1000, "Maximum number of allowed addresses (For external storages, table functions, etc).", 0) \ + M(UInt64, odbc_bridge_connection_pool_size, 16, "Connection pool size for each connection settings string in ODBC bridge.", 0) \ + M(Bool, odbc_bridge_use_connection_pooling, true, "Use connection pooling in ODBC bridge. If set to false, a new connection is created every time", 0) \ + \ + M(Seconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \ + M(UInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up an incredible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \ + M(UInt64, distributed_replica_max_ignored_errors, 0, "Number of errors that will be ignored while choosing replicas", 0) \ + \ + M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ + \ + M(DefaultTableEngine, default_temporary_table_engine, DefaultTableEngine::Memory, "Default table engine used when ENGINE is not set in CREATE TEMPORARY statement.",0) \ + M(DefaultTableEngine, default_table_engine, DefaultTableEngine::MergeTree, "Default table engine used when ENGINE is not set in CREATE statement.",0) \ + M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ + M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \ + M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ + M(Bool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ + M(Bool, optimize_trivial_approximate_count_query, false, "Use an approximate value for trivial count optimization of storages that support such estimations.", 0) \ + M(Bool, optimize_count_from_files, true, "Optimize counting rows from files in supported input formats", 0) \ + M(Bool, use_cache_for_count_from_files, true, "Use cache to count the number of rows in files", 0) \ + M(Bool, optimize_respect_aliases, true, "If it is set to true, it will respect aliases in WHERE/GROUP BY/ORDER BY, that will help with partition pruning/secondary indexes/optimize_aggregation_in_order/optimize_read_in_order/optimize_trivial_count", 0) \ + M(UInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ + M(Bool, enable_lightweight_delete, true, "Enable lightweight DELETE mutations for mergetree tables.", 0) ALIAS(allow_experimental_lightweight_delete) \ + M(UInt64, lightweight_deletes_sync, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes", 0) \ + M(Bool, apply_deleted_mask, true, "Enables filtering out rows deleted with lightweight DELETE. If disabled, a query will be able to read those rows. This is useful for debugging and \"undelete\" scenarios", 0) \ + M(Bool, optimize_normalize_count_variants, true, "Rewrite aggregate functions that semantically equals to count() as count().", 0) \ + M(Bool, optimize_injective_functions_inside_uniq, true, "Delete injective functions of one argument inside uniq*() functions.", 0) \ + M(Bool, rewrite_count_distinct_if_with_count_distinct_implementation, false, "Rewrite countDistinctIf with count_distinct_implementation configuration", 0) \ + M(Bool, convert_query_to_cnf, false, "Convert SELECT query to CNF", 0) \ + M(Bool, optimize_or_like_chain, false, "Optimize multiple OR LIKE into multiMatchAny. This optimization should not be enabled by default, because it defies index analysis in some cases.", 0) \ + M(Bool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \ + M(Bool, optimize_redundant_functions_in_order_by, true, "Remove functions from ORDER BY if its argument is also in ORDER BY", 0) \ + M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ + M(Bool, optimize_multiif_to_if, true, "Replace 'multiIf' with only one condition to 'if'.", 0) \ + M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \ + M(Bool, optimize_functions_to_subcolumns, true, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ + M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ + M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ + M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \ + M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \ + M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \ + M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \ + M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views. Use true to always deduplicate in dependent tables.", 0) \ + M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \ + M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \ + M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \ + M(Bool, allow_materialized_view_with_bad_select, true, "Allow CREATE MATERIALIZED VIEW with SELECT query that references nonexistent tables or columns. It must still be syntactically valid. Doesn't apply to refreshable MVs. Doesn't apply if the MV schema needs to be inferred from the SELECT query (i.e. if the CREATE has no column list and no TO table). Can be used for creating MV before its source table.", 0) \ + M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \ + M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \ + M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \ + M(UInt64, max_parser_backtracks, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, "Maximum parser backtracking (how many times it tries different alternatives in the recursive descend parsing process).", 0) \ + M(UInt64, max_recursive_cte_evaluation_depth, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth", 0) \ + M(Bool, allow_settings_after_format_in_insert, false, "Allow SETTINGS after FORMAT, but note, that this is not always safe (note: this is a compatibility setting).", 0) \ + M(Seconds, periodic_live_view_refresh, 60, "Interval after which periodically refreshed live view is forced to refresh.", 0) \ + M(Bool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \ + M(Bool, allow_nondeterministic_mutations, false, "Allow non-deterministic functions in ALTER UPDATE/ALTER DELETE statements", 0) \ + M(Seconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \ + M(Bool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ + M(String, function_implementation, "", "Choose function implementation for specific target or variant (experimental). If empty enable all of them.", 0) \ + M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ + M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \ + M(Bool, cast_ipv4_ipv6_default_on_conversion_error, false, "CAST operator into IPv4, CAST operator into IPV6 type, toIPv4, toIPv6 functions will return default value instead of throwing exception on conversion error.", 0) \ + M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ + M(Bool, system_events_show_zero_values, false, "When querying system.events or system.metrics tables, include all metrics, even with zero values.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, MySQLDataTypesSupportList{}, "Defines how MySQL types are converted to corresponding ClickHouse types. A comma separated list in any combination of 'decimal', 'datetime64', 'date2Date32' or 'date2String'. decimal: convert NUMERIC and DECIMAL types to Decimal when precision allows it. datetime64: convert DATETIME and TIMESTAMP types to DateTime64 instead of DateTime when precision is not 0. date2Date32: convert DATE to Date32 instead of Date. Takes precedence over date2String. date2String: convert DATE to String instead of Date. Overridden by datetime64.", 0) \ + M(Bool, optimize_trivial_insert_select, false, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ + M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ + M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \ + M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \ + M(Bool, optimize_syntax_fuse_functions, false, "Allow apply fuse aggregating function. Available only with `allow_experimental_analyzer`", 0) \ + M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \ + M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \ + M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \ + M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \ + M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \ + M(Bool, optimize_use_projections, true, "Automatically choose projections to perform SELECT query", 0) ALIAS(allow_experimental_projection_optimization) \ + M(Bool, optimize_use_implicit_projections, true, "Automatically choose implicit projections to perform SELECT query", 0) \ + M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \ + M(String, force_optimize_projection_name, "", "If it is set to a non-empty string, check that this projection is used in the query at least once.", 0) \ + M(String, preferred_optimize_projection_name, "", "If it is set to a non-empty string, ClickHouse tries to apply specified projection", 0) \ + M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ + M(Bool, async_query_sending_for_remote, true, "Asynchronously create connections and send query to shards in remote query", 0) \ + M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \ + M(Bool, describe_extend_object_types, false, "Deduce concrete type of columns of type Object in DESCRIBE query", 0) \ + M(Bool, describe_include_subcolumns, false, "If true, subcolumns of all table columns will be included into result of DESCRIBE query", 0) \ + M(Bool, describe_include_virtual_columns, false, "If true, virtual columns of table will be included into result of DESCRIBE query", 0) \ + M(Bool, describe_compact_output, false, "If true, include only column names and types into result of DESCRIBE query", 0) \ + M(Bool, apply_mutations_on_fly, false, "Only available in ClickHouse Cloud", 0) \ + M(Bool, mutations_execute_nondeterministic_on_initiator, false, "If true nondeterministic function are executed on initiator and replaced to literals in UPDATE and DELETE queries", 0) \ + M(Bool, mutations_execute_subqueries_on_initiator, false, "If true scalar subqueries are executed on initiator and replaced to literals in UPDATE and DELETE queries", 0) \ + M(UInt64, mutations_max_literal_size_to_replace, 16384, "The maximum size of serialized literal in bytes to replace in UPDATE and DELETE queries", 0) \ + \ + M(Float, create_replicated_merge_tree_fault_injection_probability, 0.0f, "The probability of a fault injection during table creation after creating metadata in ZooKeeper", 0) \ + \ + M(Bool, use_query_cache, false, "Enable the query cache", 0) \ + M(Bool, enable_writes_to_query_cache, true, "Enable storing results of SELECT queries in the query cache", 0) \ + M(Bool, enable_reads_from_query_cache, true, "Enable reading results of SELECT queries from the query cache", 0) \ + M(QueryCacheNondeterministicFunctionHandling, query_cache_nondeterministic_function_handling, QueryCacheNondeterministicFunctionHandling::Throw, "How the query cache handles queries with non-deterministic functions, e.g. now()", 0) \ + M(QueryCacheSystemTableHandling, query_cache_system_table_handling, QueryCacheSystemTableHandling::Throw, "How the query cache handles queries against system tables, i.e. tables in databases 'system.*' and 'information_schema.*'", 0) \ + M(UInt64, query_cache_max_size_in_bytes, 0, "The maximum amount of memory (in bytes) the current user may allocate in the query cache. 0 means unlimited. ", 0) \ + M(UInt64, query_cache_max_entries, 0, "The maximum number of query results the current user may store in the query cache. 0 means unlimited.", 0) \ + M(UInt64, query_cache_min_query_runs, 0, "Minimum number a SELECT query must run before its result is stored in the query cache", 0) \ + M(Milliseconds, query_cache_min_query_duration, 0, "Minimum time in milliseconds for a query to run for its result to be stored in the query cache.", 0) \ + M(Bool, query_cache_compress_entries, true, "Compress cache entries.", 0) \ + M(Bool, query_cache_squash_partial_results, true, "Squash partial result blocks to blocks of size 'max_block_size'. Reduces performance of inserts into the query cache but improves the compressability of cache entries.", 0) \ + M(Seconds, query_cache_ttl, 60, "After this time in seconds entries in the query cache become stale", 0) \ + M(Bool, query_cache_share_between_users, false, "Allow other users to read entry in the query cache", 0) \ + M(String, query_cache_tag, "", "A string which acts as a label for query cache entries. The same queries with different tags are considered different by the query cache.", 0) \ + M(Bool, enable_sharing_sets_for_mutations, true, "Allow sharing set objects build for IN subqueries between different tasks of the same mutation. This reduces memory usage and CPU consumption", 0) \ + \ + M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ + M(Bool, optimize_rewrite_aggregate_function_with_if, true, "Rewrite aggregate functions with if expression as argument when logically equivalent. For example, avg(if(cond, col, null)) can be rewritten to avgIf(cond, col)", 0) \ + M(Bool, optimize_rewrite_array_exists_to_has, false, "Rewrite arrayExists() functions to has() when logically equivalent. For example, arrayExists(x -> x = 1, arr) can be rewritten to has(arr, 1)", 0) \ + M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \ + \ + M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ + M(UInt64, max_size_to_preallocate_for_aggregation, 100'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before aggregation", 0) \ + \ + M(Bool, collect_hash_table_stats_during_joins, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ + M(UInt64, max_size_to_preallocate_for_joins, 100'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before join", 0) \ + \ + M(Bool, kafka_disable_num_consumers_limit, false, "Disable limit on kafka_num_consumers that depends on the number of available CPU cores", 0) \ + M(Bool, allow_experimental_kafka_offsets_storage_in_keeper, false, "Allow experimental feature to store Kafka related offsets in ClickHouse Keeper. When enabled a ClickHouse Keeper path and replica name can be specified to the Kafka table engine. As a result instead of the regular Kafka engine, a new type of storage engine will be used that stores the committed offsets primarily in ClickHouse Keeper", 0) \ + M(Bool, enable_software_prefetch_in_aggregation, true, "Enable use of software prefetch in aggregation", 0) \ + M(Bool, allow_aggregate_partitions_independently, false, "Enable independent aggregation of partitions on separate threads when partition key suits group by key. Beneficial when number of partitions close to number of cores and partitions have roughly the same size", 0) \ + M(Bool, force_aggregate_partitions_independently, false, "Force the use of optimization when it is applicable, but heuristics decided not to use it", 0) \ + M(UInt64, max_number_of_partitions_for_independent_aggregation, 128, "Maximal number of partitions in table to apply optimization", 0) \ + M(Float, min_hit_rate_to_use_consecutive_keys_optimization, 0.5, "Minimal hit rate of a cache which is used for consecutive keys optimization in aggregation to keep it enabled", 0) \ + \ + M(Bool, engine_file_empty_if_not_exists, false, "Allows to select data from a file engine table without file", 0) \ + M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ + M(Bool, engine_file_allow_create_multiple_files, false, "Enables or disables creating a new file on each insert in file engine tables if format has suffix.", 0) \ + M(Bool, engine_file_skip_empty_files, false, "Allows to skip empty files in file table engine", 0) \ + M(Bool, engine_url_skip_empty_files, false, "Allows to skip empty files in the URL table engine", 0) \ + M(Bool, enable_url_encoding, true, " Allows to enable/disable decoding/encoding path in URI in the URL table engine", 0) \ + M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ + M(Bool, database_replicated_enforce_synchronous_settings, false, "Enforces synchronous waiting for some queries (see also database_atomic_wait_for_drop_and_detach_synchronously, mutation_sync, alter_sync). Not recommended to enable these settings.", 0) \ + M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \ + M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \ + M(Bool, database_replicated_allow_only_replicated_engine, false, "Allow to create only Replicated tables in database with engine Replicated", 0) \ + M(UInt64, database_replicated_allow_replicated_engine_arguments, 0, "0 - Don't allow to explicitly specify ZooKeeper path and replica name for *MergeTree tables in Replicated databases. 1 - Allow. 2 - Allow, but ignore the specified path and use default one instead.", 0) \ + M(UInt64, database_replicated_allow_explicit_uuid, 0, "0 - Don't allow to explicitly specify UUIDs for tables in Replicated databases. 1 - Allow. 2 - Allow, but ignore the specified UUID and generate a random one instead.", 0) \ + M(Bool, database_replicated_allow_heavy_create, false, "Allow long-running DDL queries (CREATE AS SELECT and POPULATE) in Replicated database engine. Note that it can block DDL queue for a long time.", 0) \ + M(Bool, cloud_mode, false, "Only available in ClickHouse Cloud", 0) \ + M(UInt64, cloud_mode_engine, 1, "Only available in ClickHouse Cloud", 0) \ + M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw', 'none_only_active', 'throw_only_active', 'null_status_on_timeout_only_active'", 0) \ + M(UInt64, distributed_ddl_entry_format_version, 5, "Compatibility version of distributed DDL (ON CLUSTER) queries", 0) \ + \ + M(UInt64, external_storage_max_read_rows, 0, "Limit maximum number of rows when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializedMySQL. If equal to 0, this setting is disabled", 0) \ + M(UInt64, external_storage_max_read_bytes, 0, "Limit maximum number of bytes when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializedMySQL. If equal to 0, this setting is disabled", 0) \ + M(UInt64, external_storage_connect_timeout_sec, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connect timeout in seconds. Now supported only for MySQL", 0) \ + M(UInt64, external_storage_rw_timeout_sec, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "Read/write timeout in seconds. Now supported only for MySQL", 0) \ + \ + M(SetOperationMode, union_default_mode, SetOperationMode::Unspecified, "Set default mode in UNION query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without mode will throw exception.", 0) \ + M(SetOperationMode, intersect_default_mode, SetOperationMode::ALL, "Set default mode in INTERSECT query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without mode will throw exception.", 0) \ + M(SetOperationMode, except_default_mode, SetOperationMode::ALL, "Set default mode in EXCEPT query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without mode will throw exception.", 0) \ + M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ + M(Bool, optimize_injective_functions_in_group_by, true, "Replaces injective functions by it's arguments in GROUP BY section", 0) \ + M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ + M(Bool, optimize_group_by_constant_keys, true, "Optimize GROUP BY when all keys in block are constant", 0) \ + M(Bool, legacy_column_name_of_tuple_literal, false, "List all names of element of large tuple literals in their column names instead of hash. This settings exists only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher.", 0) \ + M(Bool, enable_named_columns_in_function_tuple, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers.", 0) \ + \ + M(Bool, query_plan_enable_optimizations, true, "Globally enable/disable query optimization at the query plan level", 0) \ + M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ + M(Bool, query_plan_lift_up_array_join, true, "Allow to move array joins up in the query plan", 0) \ + M(Bool, query_plan_push_down_limit, true, "Allow to move LIMITs down in the query plan", 0) \ + M(Bool, query_plan_split_filter, true, "Allow to split filters in the query plan", 0) \ + M(Bool, query_plan_merge_expressions, true, "Allow to merge expressions in the query plan", 0) \ + M(Bool, query_plan_merge_filters, false, "Allow to merge filters in the query plan", 0) \ + M(Bool, query_plan_filter_push_down, true, "Allow to push down filter by predicate query plan step", 0) \ + M(Bool, query_plan_convert_outer_join_to_inner_join, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values", 0) \ + M(Bool, query_plan_optimize_prewhere, true, "Allow to push down filter to PREWHERE expression for supported storages", 0) \ + M(Bool, query_plan_execute_functions_after_sorting, true, "Allow to re-order functions after sorting", 0) \ + M(Bool, query_plan_reuse_storage_ordering_for_window_functions, true, "Allow to use the storage sorting for window functions", 0) \ + M(Bool, query_plan_lift_up_union, true, "Allow to move UNIONs up so that more parts of the query plan can be optimized", 0) \ + M(Bool, query_plan_read_in_order, true, "Use query plan for read-in-order optimization", 0) \ + M(Bool, query_plan_aggregation_in_order, true, "Use query plan for aggregation-in-order optimization", 0) \ + M(Bool, query_plan_remove_redundant_sorting, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \ + M(Bool, query_plan_remove_redundant_distinct, true, "Remove redundant Distinct step in query plan", 0) \ + M(Bool, query_plan_enable_multithreading_after_window_functions, true, "Enable multithreading after evaluating window functions to allow parallel stream processing", 0) \ + M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \ + \ + M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ + M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ + \ + M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function `range` per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \ + M(UInt64, function_sleep_max_microseconds_per_block, 3000000, "Maximum number of microseconds the function `sleep` is allowed to sleep for each block. If a user called it with a larger value, it throws an exception. It is a safety threshold.", 0) \ + M(UInt64, function_visible_width_behavior, 1, "The version of `visibleWidth` behavior. 0 - only count the number of code points; 1 - correctly count zero-width and combining characters, count full-width characters as two, estimate the tab width, count delete characters.", 0) \ + M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \ + \ + M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::pread, "Method of reading data from storage file, one of: read, pread, mmap. The mmap method does not apply to clickhouse-server (it's intended for clickhouse-local).", 0) \ + M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, io_uring, pread_threadpool. The 'io_uring' method is experimental and does not work for Log, TinyLog, StripeLog, File, Set and Join, and other tables with append-able files in presence of concurrent reads and writes.", 0) \ + M(String, remote_filesystem_read_method, "threadpool", "Method of reading data from remote filesystem, one of: read, threadpool.", 0) \ + M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \ + M(Bool, remote_filesystem_read_prefetch, true, "Should use prefetching when reading data from remote filesystem.", 0) \ + M(Int64, read_priority, 0, "Priority to read data from local filesystem or remote filesystem. Only supported for 'pread_threadpool' method for local filesystem and for `threadpool` method for remote filesystem.", 0) \ + M(UInt64, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \ + M(UInt64, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \ + M(UInt64, remote_read_min_bytes_for_seek, 4 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead of read with ignore.", 0) \ + M(UInt64, merge_tree_min_bytes_per_task_for_remote_reading, 2 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes to read per task.", 0) ALIAS(filesystem_prefetch_min_bytes_for_single_read_task) \ + M(Bool, merge_tree_use_const_size_tasks_for_remote_reading, true, "Whether to use constant size tasks for reading from a remote table.", 0) \ + M(Bool, merge_tree_determine_task_size_by_prewhere_columns, true, "Whether to use only prewhere columns size to determine reading task size.", 0) \ + M(UInt64, merge_tree_compact_parts_min_granules_to_multibuffer_read, 16, "Only available in ClickHouse Cloud", 0) \ + \ + M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background. If wait_for_async_insert is false, INSERT query is processed almost instantly, otherwise client will wait until data will be flushed to table", 0) \ + M(Bool, wait_for_async_insert, true, "If true wait for processing of asynchronous insertion", 0) \ + M(Seconds, wait_for_async_insert_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "Timeout for waiting for processing asynchronous insertion", 0) \ + M(UInt64, async_insert_max_data_size, 10485760, "Maximum size in bytes of unparsed data collected per query before being inserted", 0) \ + M(UInt64, async_insert_max_query_number, 450, "Maximum number of insert queries before being inserted", 0) \ + M(Milliseconds, async_insert_poll_timeout_ms, 10, "Timeout for polling data from asynchronous insert queue", 0) \ + M(Bool, async_insert_use_adaptive_busy_timeout, true, "If it is set to true, use adaptive busy timeout for asynchronous inserts", 0) \ + M(Milliseconds, async_insert_busy_timeout_min_ms, 50, "If auto-adjusting is enabled through async_insert_use_adaptive_busy_timeout, minimum time to wait before dumping collected data per query since the first data appeared. It also serves as the initial value for the adaptive algorithm", 0) \ + M(Milliseconds, async_insert_busy_timeout_max_ms, 200, "Maximum time to wait before dumping collected data per query since the first data appeared.", 0) ALIAS(async_insert_busy_timeout_ms) \ + M(Double, async_insert_busy_timeout_increase_rate, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases", 0) \ + M(Double, async_insert_busy_timeout_decrease_rate, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases", 0) \ + \ + M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ + M(UInt64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ + M(Bool, enable_filesystem_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must be done via disk config), but allows to bypass cache for some queries if intended", 0) \ + M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \ + M(Bool, enable_filesystem_cache_log, false, "Allows to record the filesystem caching log for each query", 0) \ + M(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, "Allow to use the filesystem cache in passive mode - benefit from the existing cache entries, but don't put more entries into the cache. If you set this setting for heavy ad-hoc queries and leave it disabled for short real-time queries, this will allows to avoid cache threshing by too heavy queries and to improve the overall system efficiency.", 0) \ + M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ + M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \ + M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \ + M(UInt64, filesystem_cache_segments_batch_size, 20, "Limit on size of a single batch of file segments that a read buffer can request from cache. Too low value will lead to excessive requests to cache, too large may slow down eviction from cache", 0) \ + M(UInt64, filesystem_cache_reserve_space_wait_lock_timeout_milliseconds, 1000, "Wait time to lock cache for space reservation in filesystem cache", 0) \ + M(UInt64, temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds, (10 * 60 * 1000), "Wait time to lock cache for space reservation for temporary data in filesystem cache", 0) \ + \ + M(Bool, use_page_cache_for_disks_without_file_cache, false, "Use userspace page cache for remote disks that don't have filesystem cache enabled.", 0) \ + M(Bool, read_from_page_cache_if_exists_otherwise_bypass_cache, false, "Use userspace page cache in passive mode, similar to read_from_filesystem_cache_if_exists_otherwise_bypass_cache.", 0) \ + M(Bool, page_cache_inject_eviction, false, "Userspace page cache will sometimes invalidate some pages at random. Intended for testing.", 0) \ + \ + M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ + M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \ + M(Bool, allow_prefetched_read_pool_for_remote_filesystem, true, "Prefer prefetched threadpool if all parts are on remote filesystem", 0) \ + M(Bool, allow_prefetched_read_pool_for_local_filesystem, false, "Prefer prefetched threadpool if all parts are on local filesystem", 0) \ + \ + M(UInt64, prefetch_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the prefetch buffer to read from the filesystem.", 0) \ + M(UInt64, filesystem_prefetch_step_bytes, 0, "Prefetch step in bytes. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \ + M(UInt64, filesystem_prefetch_step_marks, 0, "Prefetch step in marks. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \ + M(UInt64, filesystem_prefetch_max_memory_usage, "1Gi", "Maximum memory usage for prefetches.", 0) \ + M(UInt64, filesystem_prefetches_limit, 200, "Maximum number of prefetches. Zero means unlimited. A setting `filesystem_prefetches_max_memory_usage` is more recommended if you want to limit the number of prefetches", 0) \ + \ + M(UInt64, use_structure_from_insertion_table_in_table_functions, 2, "Use structure from insertion table instead of schema inference from data. Possible values: 0 - disabled, 1 - enabled, 2 - auto", 0) \ + \ + M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ + M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \ + M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \ + \ + M(Bool, force_remove_data_recursively_on_drop, false, "Recursively remove data on DROP query. Avoids 'Directory not empty' error, but may silently remove detached data", 0) \ + M(Bool, check_table_dependencies, true, "Check that DDL query (such as DROP TABLE or RENAME) will not break dependencies", 0) \ + M(Bool, check_referential_table_dependencies, false, "Check that DDL query (such as DROP TABLE or RENAME) will not break referential dependencies", 0) \ + M(Bool, use_local_cache_for_remote_storage, true, "Use local cache for remote storage like HDFS or S3, it's used for remote table engine only", 0) \ + \ + M(Bool, allow_unrestricted_reads_from_keeper, false, "Allow unrestricted (without condition on path) reads from system.zookeeper table, can be handy, but is not safe for zookeeper", 0) \ + M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ + M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \ + M(Bool, allow_asynchronous_read_from_io_pool_for_merge_tree, false, "Use background I/O pool to read from MergeTree tables. This setting may increase performance for I/O bound queries", 0) \ + M(UInt64, max_streams_for_merge_tree_reading, 0, "If is not zero, limit the number of reading streams for MergeTree table.", 0) \ + \ + M(Bool, force_grouping_standard_compatibility, true, "Make GROUPING function to return 1 when argument is not used as an aggregation key", 0) \ + \ + M(Bool, schema_inference_use_cache_for_file, true, "Use cache in schema inference while using file table function", 0) \ + M(Bool, schema_inference_use_cache_for_s3, true, "Use cache in schema inference while using s3 table function", 0) \ + M(Bool, schema_inference_use_cache_for_azure, true, "Use cache in schema inference while using azure table function", 0) \ + M(Bool, schema_inference_use_cache_for_hdfs, true, "Use cache in schema inference while using hdfs table function", 0) \ + M(Bool, schema_inference_use_cache_for_url, true, "Use cache in schema inference while using url table function", 0) \ + M(Bool, schema_inference_cache_require_modification_time_for_url, true, "Use schema from cache for URL with last modification time validation (for URLs with Last-Modified header)", 0) \ + \ + M(String, compatibility, "", "Changes other settings according to provided ClickHouse version. If we know that we changed some behaviour in ClickHouse by changing some settings in some version, this compatibility setting will control these settings", 0) \ + \ + M(Map, additional_table_filters, "", "Additional filter expression which would be applied after reading from specified table. Syntax: {'table1': 'expression', 'database.table2': 'expression'}", 0) \ + M(String, additional_result_filter, "", "Additional filter expression which would be applied to query result", 0) \ + \ + M(String, workload, "default", "Name of workload to be used to access resources", 0) \ + M(Milliseconds, storage_system_stack_trace_pipe_read_timeout_ms, 100, "Maximum time to read from a pipe for receiving information from the threads when querying the `system.stack_trace` table. This setting is used for testing purposes and not meant to be changed by users.", 0) \ + \ + M(String, rename_files_after_processing, "", "Rename successfully processed files according to the specified pattern; Pattern can include the following placeholders: `%a` (full original file name), `%f` (original filename without extension), `%e` (file extension with dot), `%t` (current timestamp in µs), and `%%` (% sign)", 0) \ + \ + M(Bool, parallelize_output_from_storages, true, "Parallelize output for reading step from storage. It allows parallelization of query processing right after reading from storage if possible", 0) \ + M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ + M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ + M(Bool, throw_if_no_data_to_insert, true, "Allows or forbids empty INSERTs, enabled by default (throws an error on an empty insert)", 0) \ + M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ + M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ + M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \ + M(UInt64, keeper_max_retries, 10, "Max retries for general keeper operations", 0) \ + M(UInt64, keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for general keeper operations", 0) \ + M(UInt64, keeper_retry_max_backoff_ms, 5000, "Max backoff timeout for general keeper operations", 0) \ + M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \ + M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \ + M(UInt64, insert_keeper_retry_max_backoff_ms, 10000, "Max backoff timeout for keeper operations during insert", 0) \ + M(Float, insert_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during insert. Valid value is in interval [0.0f, 1.0f]", 0) \ + M(UInt64, insert_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ + M(Bool, force_aggregation_in_order, false, "The setting is used by the server itself to support distributed queries. Do not change it manually, because it will break normal operations. (Forces use of aggregation in order on remote nodes during distributed aggregation).", IMPORTANT) \ + M(UInt64, http_max_request_param_data_size, 10_MiB, "Limit on size of request data used as a query parameter in predefined HTTP requests.", 0) \ + M(Bool, function_json_value_return_type_allow_nullable, false, "Allow function JSON_VALUE to return nullable type.", 0) \ + M(Bool, function_json_value_return_type_allow_complex, false, "Allow function JSON_VALUE to return complex type, such as: struct, array, map.", 0) \ + M(Bool, use_with_fill_by_sorting_prefix, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently", 0) \ + M(Bool, optimize_uniq_to_count, true, "Rewrite uniq and its variants(except uniqUpTo) to count if subquery has distinct or group by clause.", 0) \ + M(Bool, use_variant_as_common_type, false, "Use Variant as a result type for if/multiIf in case when there is no common type for arguments", 0) \ + M(Bool, enable_order_by_all, true, "Enable sorting expression ORDER BY ALL.", 0) \ + M(Float, ignore_drop_queries_probability, 0, "If enabled, server will ignore all DROP table queries with specified probability (for Memory and JOIN engines it will replcase DROP to TRUNCATE). Used for testing purposes", 0) \ + M(Bool, traverse_shadow_remote_data_paths, false, "Traverse shadow directory when query system.remote_data_paths", 0) \ + M(Bool, geo_distance_returns_float64_on_float64_arguments, true, "If all four arguments to `geoDistance`, `greatCircleDistance`, `greatCircleAngle` functions are Float64, return Float64 and use double precision for internal calculations. In previous ClickHouse versions, the functions always returned Float32.", 0) \ + M(Bool, allow_get_client_http_header, false, "Allow to use the function `getClientHTTPHeader` which lets to obtain a value of an the current HTTP request's header. It is not enabled by default for security reasons, because some headers, such as `Cookie`, could contain sensitive info. Note that the `X-ClickHouse-*` and `Authentication` headers are always restricted and cannot be obtained with this function.", 0) \ + M(Bool, cast_string_to_dynamic_use_inference, false, "Use types inference during String to Dynamic conversion", 0) \ + M(Bool, enable_blob_storage_log, true, "Write information about blob storage operations to system.blob_storage_log table", 0) \ + M(Bool, use_json_alias_for_old_object_type, false, "When enabled, JSON type alias will create old experimental Object type instead of a new JSON type", 0) \ + M(Bool, allow_create_index_without_type, false, "Allow CREATE INDEX query without TYPE. Query will be ignored. Made for SQL compatibility tests.", 0) \ + M(Bool, create_index_ignore_unique, false, "Ignore UNIQUE keyword in CREATE UNIQUE INDEX. Made for SQL compatibility tests.", 0) \ + M(Bool, print_pretty_type_names, true, "Print pretty type names in the DESCRIBE query and `toTypeName` function, as well as in the `SHOW CREATE TABLE` query and the `formatQuery` function.", 0) \ + M(Bool, create_table_empty_primary_key_by_default, false, "Allow to create *MergeTree tables with empty primary key when ORDER BY and PRIMARY KEY not specified", 0) \ + M(Bool, allow_named_collection_override_by_default, true, "Allow named collections' fields override by default.", 0) \ + M(SQLSecurityType, default_normal_view_sql_security, SQLSecurityType::INVOKER, "Allows to set a default value for SQL SECURITY option when creating a normal view.", 0) \ + M(SQLSecurityType, default_materialized_view_sql_security, SQLSecurityType::DEFINER, "Allows to set a default value for SQL SECURITY option when creating a materialized view.", 0) \ + M(String, default_view_definer, "CURRENT_USER", "Allows to set a default value for DEFINER option when creating view.", 0) \ + M(UInt64, cache_warmer_threads, 4, "Only available in ClickHouse Cloud. Number of background threads for speculatively downloading new data parts into file cache, when cache_populated_by_fetch is enabled. Zero to disable.", 0) \ + M(Int64, ignore_cold_parts_seconds, 0, "Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.", 0) \ + M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \ + M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \ + M(Bool, allow_deprecated_error_prone_window_functions, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)", 0) \ + M(Bool, allow_deprecated_snowflake_conversion_functions, false, "Enables deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake.", 0) \ + M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ + M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \ + M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \ + M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \ + M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \ + M(Bool, create_if_not_exists, false, "Enable IF NOT EXISTS for CREATE statements by default", 0) \ + \ + \ + /* ###################################### */ \ + /* ######## EXPERIMENTAL FEATURES ####### */ \ + /* ###################################### */ \ + M(Bool, allow_experimental_materialized_postgresql_table, false, "Allows to use the MaterializedPostgreSQL table engine. Disabled by default, because this feature is experimental", 0) \ + M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ + M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \ + M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions", 0) \ + M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \ + M(Bool, allow_experimental_time_series_table, false, "Allows experimental TimeSeries table engine", 0) \ + M(Bool, allow_experimental_vector_similarity_index, false, "Allow experimental vector similarity index", 0) \ + M(Bool, allow_experimental_variant_type, false, "Allow Variant data type", 0) \ + M(Bool, allow_experimental_dynamic_type, false, "Allow Dynamic data type", 0) \ + M(Bool, allow_experimental_json_type, false, "Allow JSON data type", 0) \ + M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ + M(UInt64, max_limit_for_ann_queries, 1'000'000, "SELECT queries with LIMIT bigger than this setting cannot use ANN indexes. Helps to prevent memory overflows in ANN search indexes.", 0) \ + M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \ + M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \ + M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ + M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ + M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ + M(Int32, join_to_sort_minimum_perkey_rows, 40, "The lower limit of per-key average rows in the right table to determine whether to rerange the right table by key in left or inner join. This setting ensures that the optimization is not applied for sparse table keys", 0) \ + M(Int32, join_to_sort_maximum_table_rows, 10000, "The maximum number of rows in the right table to determine whether to rerange the right table by key in left or inner join.", 0) \ + M(Bool, allow_experimental_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join.", 0) \ + M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ + M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ + \ + M(Bool, allow_statistics_optimize, false, "Allows using statistics to optimize queries", 0) ALIAS(allow_statistic_optimize) \ + M(Bool, allow_experimental_statistics, false, "Allows using statistics", 0) ALIAS(allow_experimental_statistic) \ + \ + /* Parallel replicas */ \ + M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ + M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ + M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ + M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \ + M(UInt64, parallel_replicas_custom_key_range_lower, 0, "Lower bound for the universe that the parallel replicas custom range filter is calculated over", 0) \ + M(UInt64, parallel_replicas_custom_key_range_upper, 0, "Upper bound for the universe that the parallel replicas custom range filter is calculated over. A value of 0 disables the upper bound, setting it to the max value of the custom key expression", 0) \ + M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \ + M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \ + M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \ + M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \ + M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \ + M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \ + M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \ + M(UInt64, parallel_replicas_mark_segment_size, 0, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384]", 0) \ + M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as ' :: ' if archive has correct extension", 0) \ + M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \ + \ + M(Bool, allow_experimental_inverted_index, false, "If it is set to true, allow to use experimental inverted index.", 0) \ + M(Bool, allow_experimental_full_text_index, false, "If it is set to true, allow to use experimental full-text index.", 0) \ + \ + M(Bool, allow_experimental_join_condition, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y.", 0) \ + \ + M(Bool, allow_experimental_analyzer, true, "Allow new query analyzer.", IMPORTANT) ALIAS(enable_analyzer) \ + M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \ + \ + M(Bool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \ + M(Seconds, live_view_heartbeat_interval, 15, "The heartbeat interval in seconds to indicate live query is alive.", 0) \ + M(UInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.", 0) \ + \ + M(Bool, allow_experimental_window_view, false, "Enable WINDOW VIEW. Not mature enough.", 0) \ + M(Seconds, window_view_clean_interval, 60, "The clean interval of window view in seconds to free outdated data.", 0) \ + M(Seconds, window_view_heartbeat_interval, 15, "The heartbeat interval in seconds to indicate watch query is alive.", 0) \ + M(Seconds, wait_for_window_view_fire_signal_timeout, 10, "Timeout for waiting for window view fire signal in event time processing", 0) \ + \ + M(Bool, allow_experimental_refreshable_materialized_view, false, "Allow refreshable materialized views (CREATE MATERIALIZED VIEW REFRESH ...).", 0) \ + M(Bool, stop_refreshable_materialized_views_on_startup, false, "On server startup, prevent scheduling of refreshable materialized views, as if with SYSTEM STOP VIEWS. You can manually start them with SYSTEM START VIEWS or SYSTEM START VIEW afterwards. Also applies to newly created views. Has no effect on non-refreshable materialized views.", 0) \ + \ + M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \ + M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \ + \ + /** Experimental feature for moving data between shards. */ \ + M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \ + + /** End of experimental features */ + +// End of COMMON_SETTINGS +// Please add settings related to formats in FormatFactorySettingsDeclaration.h, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. + +#define OBSOLETE_SETTINGS(M, ALIAS) \ + /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \ + MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \ + MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \ + MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_database_atomic, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_bigint_types, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_window_functions, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_geo_types, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_query_cache, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_alter_materialized_view_structure, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_shared_merge_tree, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_database_replicated, true) \ + \ + MAKE_OBSOLETE(M, Milliseconds, async_insert_stale_timeout_ms, 0) \ + MAKE_OBSOLETE(M, StreamingHandleErrorMode, handle_kafka_error_mode, StreamingHandleErrorMode::DEFAULT) \ + MAKE_OBSOLETE(M, Bool, database_replicated_ddl_output, true) \ + MAKE_OBSOLETE(M, UInt64, replication_alter_columns_timeout, 60) \ + MAKE_OBSOLETE(M, UInt64, odbc_max_field_size, 0) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_map_type, true) \ + MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_temporary_directories_interval_seconds, 60) \ + MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_parts_interval_seconds, 1) \ + MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \ + MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \ + MAKE_OBSOLETE(M, Bool, use_mysql_types_in_show_columns, false) \ + MAKE_OBSOLETE(M, Bool, s3queue_allow_experimental_sharded_mode, false) \ + /* moved to config.xml: see also src/Core/ServerSettings.h */ \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_buffer_flush_schedule_pool_size, 16) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_pool_size, 16) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, Float, background_merges_mutations_concurrency_ratio, 2) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_move_pool_size, 8) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_fetches_pool_size, 8) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_common_pool_size, 8) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_schedule_pool_size, 128) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_message_broker_schedule_pool_size, 16) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_distributed_schedule_pool_size, 16) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_remote_read_network_bandwidth_for_server, 0) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_remote_write_network_bandwidth_for_server, 0) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, async_insert_threads, 16) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_fetches_network_bandwidth_for_server, 0) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_sends_network_bandwidth_for_server, 0) \ + MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_entries_for_hash_table_stats, 10'000) \ + /* ---- */ \ + MAKE_OBSOLETE(M, DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic) \ + MAKE_OBSOLETE(M, UInt64, max_pipeline_depth, 0) \ + MAKE_OBSOLETE(M, Seconds, temporary_live_view_timeout, 1) \ + MAKE_OBSOLETE(M, Milliseconds, async_insert_cleanup_timeout_ms, 1000) \ + MAKE_OBSOLETE(M, Bool, optimize_fuse_sum_count_avg, 0) \ + MAKE_OBSOLETE(M, Seconds, drain_timeout, 3) \ + MAKE_OBSOLETE(M, UInt64, backup_threads, 16) \ + MAKE_OBSOLETE(M, UInt64, restore_threads, 16) \ + MAKE_OBSOLETE(M, Bool, optimize_duplicate_order_by_and_distinct, false) \ + MAKE_OBSOLETE(M, UInt64, parallel_replicas_min_number_of_granules_to_enable, 0) \ + MAKE_OBSOLETE(M, Bool, query_plan_optimize_projection, true) \ + MAKE_OBSOLETE(M, Bool, query_cache_store_results_of_queries_with_nondeterministic_functions, false) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_annoy_index, false) \ + MAKE_OBSOLETE(M, UInt64, max_threads_for_annoy_index_creation, 4) \ + MAKE_OBSOLETE(M, Int64, annoy_index_search_k_nodes, -1) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_usearch_index, false) \ + MAKE_OBSOLETE(M, Bool, optimize_move_functions_out_of_any, false) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_undrop_table_query, true) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_s3queue, true) \ + MAKE_OBSOLETE(M, Bool, query_plan_optimize_primary_key, true) \ + MAKE_OBSOLETE(M, Bool, optimize_monotonous_functions_in_order_by, false) \ + MAKE_OBSOLETE(M, UInt64, http_max_chunk_size, 100_GiB) \ + + /** The section above is for obsolete settings. Do not add anything there. */ +#endif /// __CLION_IDE__ + + +#define LIST_OF_SETTINGS(M, ALIAS) \ + COMMON_SETTINGS(M, ALIAS) \ + OBSOLETE_SETTINGS(M, ALIAS) \ + FORMAT_FACTORY_SETTINGS(M, ALIAS) \ + OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \ + +DECLARE_SETTINGS_TRAITS_ALLOW_CUSTOM_SETTINGS(SettingsTraits, LIST_OF_SETTINGS) + + +/** Settings of query execution. + * These settings go to users.xml. + */ +struct SettingsImpl : public BaseSettings, public IHints<2> +{ + SettingsImpl() = default; + + /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings). + * The profile can also be set using the `set` functions, like the profile setting. + */ + void setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config); + + /// Load settings from configuration file, at "path" prefix in configuration. + void loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config); + + /// Dumps profile events to column of type Map(String, String) + void dumpToMapColumn(IColumn * column, bool changed_only = true); + + /// Check that there is no user-level settings at the top level in config. + /// This is a common source of mistake (user don't know where to write user-level setting). + static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path); + + std::vector getAllRegisteredNames() const override; + + void set(std::string_view name, const Field & value) override; + +private: + void applyCompatibilitySetting(const String & compatibility); + + std::unordered_set settings_changed_by_compatibility_setting; +}; + + IMPLEMENT_SETTINGS_TRAITS(SettingsTraits, LIST_OF_SETTINGS) /** Set the settings from the profile (in the server configuration, many settings can be listed in one profile). * The profile can also be set using the `set` functions, like the `profile` setting. */ -void Settings::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config) +void SettingsImpl::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config) { String elem = "profiles." + profile_name; @@ -44,7 +1137,7 @@ void Settings::setProfile(const String & profile_name, const Poco::Util::Abstrac } } -void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config) +void SettingsImpl::loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config) { if (!config.has(path)) throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no path '{}' in configuration file.", path); @@ -58,7 +1151,7 @@ void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::Abs } } -void Settings::dumpToMapColumn(IColumn * column, bool changed_only) +void SettingsImpl::dumpToMapColumn(IColumn * column, bool changed_only) { /// Convert ptr and make simple check auto * column_map = column ? &typeid_cast(*column) : nullptr; @@ -82,12 +1175,12 @@ void Settings::dumpToMapColumn(IColumn * column, bool changed_only) offsets.push_back(offsets.back() + size); } -void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path) +void SettingsImpl::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path) { if (config.getBool("skip_check_for_incorrect_settings", false)) return; - Settings settings; + SettingsImpl settings; for (const auto & setting : settings.all()) { const auto & name = setting.getName(); @@ -104,7 +1197,7 @@ void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfigura } } -std::vector Settings::getAllRegisteredNames() const +std::vector SettingsImpl::getAllRegisteredNames() const { std::vector all_settings; for (const auto & setting_field : all()) @@ -112,7 +1205,7 @@ std::vector Settings::getAllRegisteredNames() const return all_settings; } -void Settings::set(std::string_view name, const Field & value) +void SettingsImpl::set(std::string_view name, const Field & value) { if (name == "compatibility") { @@ -130,7 +1223,7 @@ void Settings::set(std::string_view name, const Field & value) BaseSettings::set(name, value); } -void Settings::applyCompatibilitySetting(const String & compatibility_value) +void SettingsImpl::applyCompatibilitySetting(const String & compatibility_value) { /// First, revert all changes applied by previous compatibility setting for (const auto & setting_name : settings_changed_by_compatibility_setting) @@ -163,6 +1256,279 @@ void Settings::applyCompatibilitySetting(const String & compatibility_value) } } -IMPLEMENT_SETTINGS_TRAITS(FormatFactorySettingsTraits, LIST_OF_ALL_FORMAT_SETTINGS) +#define INITIALIZE_SETTING_EXTERN(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) \ + Settings ## TYPE NAME = & Settings ## Impl :: NAME; + +namespace Setting +{ + LIST_OF_SETTINGS(INITIALIZE_SETTING_EXTERN, SKIP_ALIAS) +} + +#undef INITIALIZE_SETTING_EXTERN + +Settings::Settings() + : impl(std::make_unique()) +{} + +Settings::Settings(const Settings & settings) + : impl(std::make_unique(*settings.impl)) +{} + +Settings::Settings(Settings && settings) noexcept + : impl(std::make_unique(std::move(*settings.impl))) +{} + +Settings::~Settings() = default; + +Settings & Settings::operator=(const Settings & other) +{ + *impl = *other.impl; + return *this; +} + +bool Settings::operator==(const Settings & other) const +{ + return *impl == *other.impl; +} + +#define IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR(CLASS_NAME, TYPE) \ + const SettingField##TYPE & Settings::operator[](CLASS_NAME##TYPE t) const \ + { \ + return impl.get()->*t; \ + } \ + SettingField##TYPE & Settings::operator[](CLASS_NAME##TYPE t) \ + { \ + return impl.get()->*t; \ + } + +COMMON_SETTINGS_SUPPORTED_TYPES(Settings, IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR) +#undef IMPLEMENT_SETTING_SUBSCRIPT_OPERATOR + +bool Settings::has(std::string_view name) const +{ + return impl->has(name); +} + +bool Settings::isChanged(std::string_view name) const +{ + return impl->isChanged(name); +} + +bool Settings::tryGet(std::string_view name, Field & value) const +{ + return impl->tryGet(name, value); +} + +Field Settings::get(std::string_view name) const +{ + return impl->get(name); +} + +void Settings::set(std::string_view name, const Field & value) +{ + impl->set(name, value); +} + +void Settings::setDefaultValue(std::string_view name) +{ + impl->resetToDefault(name); +} + +std::vector Settings::getHints(const String & name) const +{ + return impl->getHints(name); +} + +String Settings::toString() const +{ + return impl->toString(); +} + +SettingsChanges Settings::changes() const +{ + return impl->changes(); +} + +void Settings::applyChanges(const SettingsChanges & changes) +{ + impl->applyChanges(changes); +} + +std::vector Settings::getAllRegisteredNames() const +{ + std::vector setting_names; + for (const auto & setting : impl->all()) + { + setting_names.emplace_back(setting.getName()); + } + return setting_names; +} + +std::vector Settings::getChangedAndObsoleteNames() const +{ + std::vector setting_names; + for (const auto & setting : impl->allChanged()) + { + if (setting.isObsolete()) + setting_names.emplace_back(setting.getName()); + } + return setting_names; +} + +std::vector Settings::getUnchangedNames() const +{ + std::vector setting_names; + for (const auto & setting : impl->allUnchanged()) + { + setting_names.emplace_back(setting.getName()); + } + return setting_names; +} + +void Settings::dumpToSystemSettingsColumns(MutableColumnsAndConstraints & params) const +{ + MutableColumns & res_columns = params.res_columns; + + const auto fill_data_for_setting = [&](std::string_view setting_name, const auto & setting) + { + res_columns[1]->insert(setting.getValueString()); + res_columns[2]->insert(setting.isValueChanged()); + res_columns[3]->insert(setting.getDescription()); + + Field min, max; + SettingConstraintWritability writability = SettingConstraintWritability::WRITABLE; + params.constraints.get(*this, setting_name, min, max, writability); + + /// These two columns can accept strings only. + if (!min.isNull()) + min = Settings::valueToStringUtil(setting_name, min); + if (!max.isNull()) + max = Settings::valueToStringUtil(setting_name, max); + + res_columns[4]->insert(min); + res_columns[5]->insert(max); + res_columns[6]->insert(writability == SettingConstraintWritability::CONST); + res_columns[7]->insert(setting.getTypeName()); + res_columns[8]->insert(setting.getDefaultValueString()); + res_columns[10]->insert(setting.isObsolete()); + }; + + const auto & settings_to_aliases = SettingsImpl::Traits::settingsToAliases(); + for (const auto & setting : impl->all()) + { + const auto & setting_name = setting.getName(); + res_columns[0]->insert(setting_name); + + fill_data_for_setting(setting_name, setting); + res_columns[9]->insert(""); + + if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end()) + { + for (const auto alias : it->second) + { + res_columns[0]->insert(alias); + fill_data_for_setting(alias, setting); + res_columns[9]->insert(setting_name); + } + } + } +} + +void Settings::dumpToMapColumn(IColumn * column, bool changed_only) const +{ + impl->dumpToMapColumn(column, changed_only); +} + +NameToNameMap Settings::toNameToNameMap() const +{ + NameToNameMap query_parameters; + for (const auto & param : *impl) + { + std::string value; + ReadBufferFromOwnString buf(param.getValueString()); + readQuoted(value, buf); + query_parameters.emplace(param.getName(), value); + } + return query_parameters; +} + +void Settings::write(WriteBuffer & out, SettingsWriteFormat format) const +{ + impl->write(out, format); +} + +void Settings::read(ReadBuffer & in, SettingsWriteFormat format) +{ + impl->read(in, format); +} + +void Settings::addToProgramOptions(boost::program_options::options_description & options) +{ + addProgramOptions(*impl, options); +} + +void Settings::addToProgramOptions(std::string_view setting_name, boost::program_options::options_description & options) +{ + const auto & accessor = SettingsImpl::Traits::Accessor::instance(); + size_t index = accessor.find(setting_name); + chassert(index != static_cast(-1)); + auto on_program_option = boost::function1( + [this, setting_name](const std::string & value) + { + this->set(setting_name, value); + }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + setting_name.data(), boost::program_options::value()->composing()->notifier(on_program_option), accessor.getDescription(index)))); // NOLINT +} + +void Settings::addToProgramOptionsAsMultitokens(boost::program_options::options_description & options) const +{ + addProgramOptionsAsMultitokens(*impl, options); +} + +void Settings::addToClientOptions(Poco::Util::LayeredConfiguration &config, const boost::program_options::variables_map &options, bool repeated_settings) const +{ + for (const auto & setting : impl->all()) + { + const auto & name = setting.getName(); + if (options.count(name)) + { + if (repeated_settings) + config.setString(name, options[name].as().back()); + else + config.setString(name, options[name].as()); + } + } +} + +Field Settings::castValueUtil(std::string_view name, const Field & value) +{ + return SettingsImpl::castValueUtil(name, value); +} + +String Settings::valueToStringUtil(std::string_view name, const Field & value) +{ + return SettingsImpl::valueToStringUtil(name, value); +} + +Field Settings::stringToValueUtil(std::string_view name, const String & str) +{ + return SettingsImpl::stringToValueUtil(name, str); +} + +bool Settings::hasBuiltin(std::string_view name) +{ + return SettingsImpl::hasBuiltin(name); +} + +std::string_view Settings::resolveName(std::string_view name) +{ + return SettingsImpl::Traits::resolveName(name); +} + +void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path) +{ + SettingsImpl::checkNoSettingNamesAtTopLevel(config, config_path); +} } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index fadc4079fe0..3c79139b27f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1,1377 +1,155 @@ #pragma once -/// CLion freezes for a minute on every keypress in any file including this. -#if !defined(__CLION_IDE__) - -#include -#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include -namespace Poco::Util +namespace boost { - class AbstractConfiguration; +namespace program_options +{ +class options_description; +class variables_map; } +} + +namespace Poco +{ +namespace Util +{ +class AbstractConfiguration; +class LayeredConfiguration; +} +} + +using NameToNameMap = std::unordered_map; namespace DB { class IColumn; +struct MutableColumnsAndConstraints; +class ReadBuffer; +struct SettingsImpl; +class WriteBuffer; -/** List of settings: type, name, default value, description, flags - * - * This looks rather inconvenient. It is done that way to avoid repeating settings in different places. - * Note: as an alternative, we could implement settings to be completely dynamic in form of map: String -> Field, - * but we are not going to do it, because settings is used everywhere as static struct fields. - * - * `flags` can be either 0 or IMPORTANT. - * A setting is "IMPORTANT" if it affects the results of queries and can't be ignored by older versions. - * - * When adding new or changing existing settings add them to settings changes history in SettingsChangesHistory.h - * for tracking settings changes in different versions and for special `compatibility` setting to work correctly. - */ - -// clang-format off -#define COMMON_SETTINGS(M, ALIAS) \ - M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\ - M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \ - M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \ - M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \ - M(UInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \ - M(UInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \ - M(UInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \ - M(UInt64, min_insert_block_size_rows_for_materialized_views, 0, "Like min_insert_block_size_rows, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_rows)", 0) \ - M(UInt64, min_insert_block_size_bytes_for_materialized_views, 0, "Like min_insert_block_size_bytes, but applied only during pushing to MATERIALIZED VIEW (default: min_insert_block_size_bytes)", 0) \ - M(UInt64, min_external_table_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough.", 0) \ - M(UInt64, min_external_table_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough.", 0) \ - M(UInt64, max_joined_block_size_rows, DEFAULT_BLOCK_SIZE, "Maximum block size for JOIN result (if join algorithm supports it). 0 means unlimited.", 0) \ - M(UInt64, max_insert_threads, 0, "The maximum number of threads to execute the INSERT SELECT query. Values 0 or 1 means that INSERT SELECT is not run in parallel. Higher values will lead to higher memory usage. Parallel INSERT SELECT has effect only if the SELECT part is run on parallel, see 'max_threads' setting.", 0) \ - M(UInt64, max_insert_delayed_streams_for_parallel_write, 0, "The maximum number of streams (columns) to delay final part flush. Default - auto (1000 in case of underlying storage supports parallel write, for example S3 and disabled otherwise)", 0) \ - M(MaxThreads, max_final_threads, 0, "The maximum number of threads to read from table with FINAL.", 0) \ - M(UInt64, max_threads_for_indexes, 0, "The maximum number of threads process indices.", 0) \ - M(MaxThreads, max_threads, 0, "The maximum number of threads to execute the request. By default, it is determined automatically.", 0) \ - M(Bool, use_concurrency_control, true, "Respect the server's concurrency control (see the `concurrent_threads_soft_limit_num` and `concurrent_threads_soft_limit_ratio_to_cores` global server settings). If disabled, it allows using a larger number of threads even if the server is overloaded (not recommended for normal usage, and needed mostly for tests).", 0) \ - M(MaxThreads, max_download_threads, 4, "The maximum number of threads to download data (e.g. for URL engine).", 0) \ - M(MaxThreads, max_parsing_threads, 0, "The maximum number of threads to parse data in input formats that support parallel parsing. By default, it is determined automatically", 0) \ - M(UInt64, max_download_buffer_size, 10*1024*1024, "The maximal size of buffer for parallel downloading (e.g. for URL engine) per each thread.", 0) \ - M(UInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ - M(UInt64, max_read_buffer_size_local_fs, 128*1024, "The maximum size of the buffer to read from local filesystem. If set to 0 then max_read_buffer_size will be used.", 0) \ - M(UInt64, max_read_buffer_size_remote_fs, 0, "The maximum size of the buffer to read from remote filesystem. If set to 0 then max_read_buffer_size will be used.", 0) \ - M(UInt64, max_distributed_connections, 1024, "The maximum number of connections for distributed processing of one query (should be greater than max_threads).", 0) \ - M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "The maximum number of bytes of a query string parsed by the SQL parser. Data in the VALUES clause of INSERT queries is processed by a separate stream parser (that consumes O(1) RAM) and not affected by this restriction.", 0) \ - M(UInt64, interactive_delay, 100000, "The interval in microseconds to check if the request is cancelled, and to send progress info.", 0) \ - M(Seconds, connect_timeout, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connection timeout if there are no replicas.", 0) \ - M(Milliseconds, handshake_timeout_ms, 10000, "Timeout for receiving HELLO packet from replicas.", 0) \ - M(Milliseconds, connect_timeout_with_failover_ms, 1000, "Connection timeout for selecting first healthy replica.", 0) \ - M(Milliseconds, connect_timeout_with_failover_secure_ms, 1000, "Connection timeout for selecting first healthy replica (for secure connections).", 0) \ - M(Seconds, receive_timeout, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "Timeout for receiving data from network, in seconds. If no bytes were received in this interval, exception is thrown. If you set this setting on client, the 'send_timeout' for the socket will be also set on the corresponding connection end on the server.", 0) \ - M(Seconds, send_timeout, DBMS_DEFAULT_SEND_TIMEOUT_SEC, "Timeout for sending data to network, in seconds. If client needs to sent some data, but it did not able to send any bytes in this interval, exception is thrown. If you set this setting on client, the 'receive_timeout' for the socket will be also set on the corresponding connection end on the server.", 0) \ - M(Seconds, tcp_keep_alive_timeout, DEFAULT_TCP_KEEP_ALIVE_TIMEOUT /* less than DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC */, "The time in seconds the connection needs to remain idle before TCP starts sending keepalive probes", 0) \ - M(Milliseconds, hedged_connection_timeout_ms, 50, "Connection timeout for establishing connection with replica for Hedged requests", 0) \ - M(Milliseconds, receive_data_timeout_ms, 2000, "Connection timeout for receiving first packet of data or packet with positive progress from replica", 0) \ - M(Bool, use_hedged_requests, true, "Use hedged requests for distributed queries", 0) \ - M(Bool, allow_changing_replica_until_first_data_packet, false, "Allow HedgedConnections to change replica until receiving first data packet", 0) \ - M(Milliseconds, queue_max_wait_ms, 0, "The wait time in the request queue, if the number of concurrent requests exceeds the maximum.", 0) \ - M(Milliseconds, connection_pool_max_wait_ms, 0, "The wait time when the connection pool is full.", 0) \ - M(Milliseconds, replace_running_query_max_wait_ms, 5000, "The wait time for running query with the same query_id to finish when setting 'replace_running_query' is active.", 0) \ - M(Milliseconds, kafka_max_wait_ms, 5000, "The wait time for reading from Kafka before retry.", 0) \ - M(Milliseconds, rabbitmq_max_wait_ms, 5000, "The wait time for reading from RabbitMQ before retry.", 0) \ - M(UInt64, poll_interval, DBMS_DEFAULT_POLL_INTERVAL, "Block at the query wait loop on the server for the specified number of seconds.", 0) \ - M(UInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \ - M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \ - M(UInt64, connections_with_failover_max_tries, 3, "The maximum number of attempts to connect to replicas.", 0) \ - M(UInt64, s3_strict_upload_part_size, S3::DEFAULT_STRICT_UPLOAD_PART_SIZE, "The exact size of part to upload during multipart upload to S3 (some implementations does not supports variable size parts).", 0) \ - M(UInt64, azure_strict_upload_part_size, 0, "The exact size of part to upload during multipart upload to Azure blob storage.", 0) \ - M(UInt64, azure_max_blocks_in_multipart_upload, 50000, "Maximum number of blocks in multipart upload for Azure.", 0) \ - M(UInt64, s3_min_upload_part_size, S3::DEFAULT_MIN_UPLOAD_PART_SIZE, "The minimum size of part to upload during multipart upload to S3.", 0) \ - M(UInt64, s3_max_upload_part_size, S3::DEFAULT_MAX_UPLOAD_PART_SIZE, "The maximum size of part to upload during multipart upload to S3.", 0) \ - M(UInt64, azure_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage.", 0) \ - M(UInt64, azure_max_upload_part_size, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage.", 0) \ - M(UInt64, s3_upload_part_size_multiply_factor, S3::DEFAULT_UPLOAD_PART_SIZE_MULTIPLY_FACTOR, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \ - M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, S3::DEFAULT_UPLOAD_PART_SIZE_MULTIPLY_PARTS_COUNT_THRESHOLD, "Each time this number of parts was uploaded to S3, s3_min_upload_part_size is multiplied by s3_upload_part_size_multiply_factor.", 0) \ - M(UInt64, s3_max_part_number, S3::DEFAULT_MAX_PART_NUMBER, "Maximum part number number for s3 upload part.", 0) \ - M(UInt64, s3_max_single_operation_copy_size, S3::DEFAULT_MAX_SINGLE_OPERATION_COPY_SIZE, "Maximum size for a single copy operation in s3", 0) \ - M(UInt64, azure_upload_part_size_multiply_factor, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage.", 0) \ - M(UInt64, azure_upload_part_size_multiply_parts_count_threshold, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor.", 0) \ - M(UInt64, s3_max_inflight_parts_for_one_file, S3::DEFAULT_MAX_INFLIGHT_PARTS_FOR_ONE_FILE, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ - M(UInt64, azure_max_inflight_parts_for_one_file, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited.", 0) \ - M(UInt64, s3_max_single_part_upload_size, S3::DEFAULT_MAX_SINGLE_PART_UPLOAD_SIZE, "The maximum size of object to upload using singlepart upload to S3.", 0) \ - M(UInt64, azure_max_single_part_upload_size, 100*1024*1024, "The maximum size of object to upload using singlepart upload to Azure blob storage.", 0) \ - M(UInt64, azure_max_single_part_copy_size, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage.", 0) \ - M(UInt64, s3_max_single_read_retries, S3::DEFAULT_MAX_SINGLE_READ_TRIES, "The maximum number of retries during single S3 read.", 0) \ - M(UInt64, azure_max_single_read_retries, 4, "The maximum number of retries during single Azure blob storage read.", 0) \ - M(UInt64, azure_max_unexpected_write_error_retries, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write", 0) \ - M(UInt64, s3_max_unexpected_write_error_retries, S3::DEFAULT_MAX_UNEXPECTED_WRITE_ERROR_RETRIES, "The maximum number of retries in case of unexpected errors during S3 write.", 0) \ - M(UInt64, s3_max_redirects, S3::DEFAULT_MAX_REDIRECTS, "Max number of S3 redirects hops allowed.", 0) \ - M(UInt64, s3_max_connections, S3::DEFAULT_MAX_CONNECTIONS, "The maximum number of connections per server.", 0) \ - M(UInt64, s3_max_get_rps, 0, "Limit on S3 GET request per second rate before throttling. Zero means unlimited.", 0) \ - M(UInt64, s3_max_get_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_get_rps`", 0) \ - M(UInt64, s3_max_put_rps, 0, "Limit on S3 PUT request per second rate before throttling. Zero means unlimited.", 0) \ - M(UInt64, s3_max_put_burst, 0, "Max number of requests that can be issued simultaneously before hitting request per second limit. By default (0) equals to `s3_max_put_rps`", 0) \ - M(UInt64, s3_list_object_keys_size, S3::DEFAULT_LIST_OBJECT_KEYS_SIZE, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_use_adaptive_timeouts, S3::DEFAULT_USE_ADAPTIVE_TIMEOUTS, "When adaptive timeouts are enabled first two attempts are made with low receive and send timeout", 0) \ - M(UInt64, azure_list_object_keys_size, 1000, "Maximum number of files that could be returned in batch by ListObject request", 0) \ - M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ - M(Bool, azure_truncate_on_insert, false, "Enables or disables truncate before insert in azure engine tables.", 0) \ - M(Bool, s3_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in s3 engine tables", 0) \ - M(Bool, s3_skip_empty_files, false, "Allow to skip empty files in s3 table engine", 0) \ - M(Bool, azure_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in azure engine tables", 0) \ - M(Bool, s3_check_objects_after_upload, false, "Check each uploaded object to s3 with head request to be sure that upload was successful", 0) \ - M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \ - M(Bool, azure_allow_parallel_part_upload, true, "Use multiple threads for azure multipart upload.", 0) \ - M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ - M(Bool, hdfs_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ - M(Bool, azure_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \ - M(Bool, s3_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in S3 table engine", 0) \ - M(Bool, hdfs_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in HDFS table engine", 0) \ - M(Bool, azure_ignore_file_doesnt_exist, false, "Return 0 rows when the requested files don't exist, instead of throwing an exception in AzureBlobStorage table engine", 0) \ - M(UInt64, azure_sdk_max_retries, 10, "Maximum number of retries in azure sdk", 0) \ - M(UInt64, azure_sdk_retry_initial_backoff_ms, 10, "Minimal backoff between retries in azure sdk", 0) \ - M(UInt64, azure_sdk_retry_max_backoff_ms, 1000, "Maximal backoff between retries in azure sdk", 0) \ - M(Bool, s3_validate_request_settings, true, "Validate S3 request settings", 0) \ - M(Bool, s3_disable_checksum, S3::DEFAULT_DISABLE_CHECKSUM, "Do not calculate a checksum when sending a file to S3. This speeds up writes by avoiding excessive processing passes on a file. It is mostly safe as the data of MergeTree tables is checksummed by ClickHouse anyway, and when S3 is accessed with HTTPS, the TLS layer already provides integrity while transferring through the network. While additional checksums on S3 give defense in depth.", 0) \ - M(UInt64, s3_retry_attempts, S3::DEFAULT_RETRY_ATTEMPTS, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries", 0) \ - M(UInt64, s3_request_timeout_ms, S3::DEFAULT_REQUEST_TIMEOUT_MS, "Idleness timeout for sending and receiving data to/from S3. Fail if a single TCP read or write call blocks for this long.", 0) \ - M(UInt64, s3_connect_timeout_ms, S3::DEFAULT_CONNECT_TIMEOUT_MS, "Connection timeout for host from s3 disks.", 0) \ - M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \ - M(String, s3queue_default_zookeeper_path, "/clickhouse/s3queue/", "Default zookeeper path prefix for S3Queue engine", 0) \ - M(Bool, s3queue_enable_logging_to_s3queue_log, false, "Enable writing to system.s3queue_log. The value can be overwritten per table with table settings", 0) \ - M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \ - M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \ - M(Bool, hdfs_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in hdfs engine tables", 0) \ - M(Bool, hdfs_skip_empty_files, false, "Allow to skip empty files in hdfs table engine", 0) \ - M(Bool, azure_skip_empty_files, false, "Allow to skip empty files in azure table engine", 0) \ - M(UInt64, hsts_max_age, 0, "Expired time for hsts. 0 means disable HSTS.", 0) \ - M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ - M(Bool, use_uncompressed_cache, false, "Whether to use the cache of uncompressed blocks.", 0) \ - M(Bool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.", 0) \ - M(UInt64, max_remote_read_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for read.", 0) \ - M(UInt64, max_remote_write_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for write.", 0) \ - M(UInt64, max_local_read_bandwidth, 0, "The maximum speed of local reads in bytes per second.", 0) \ - M(UInt64, max_local_write_bandwidth, 0, "The maximum speed of local writes in bytes per second.", 0) \ - M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams, and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \ - M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \ - M(Bool, dictionary_validate_primary_key_type, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64.", 0) \ - M(Bool, distributed_insert_skip_read_only_replicas, false, "If true, INSERT into Distributed will skip read-only replicas.", 0) \ - M(Bool, distributed_foreground_insert, false, "If setting is enabled, insert query into distributed waits until data are sent to all nodes in a cluster. \n\nEnables or disables synchronous data insertion into a `Distributed` table.\n\nBy default, when inserting data into a Distributed table, the ClickHouse server sends data to cluster nodes in the background. When `distributed_foreground_insert` = 1, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true).", 0) ALIAS(insert_distributed_sync) \ - M(UInt64, distributed_background_insert_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) ALIAS(insert_distributed_timeout) \ - M(Milliseconds, distributed_background_insert_sleep_time_ms, 100, "Sleep time for background INSERTs into Distributed, in case of any errors delay grows exponentially.", 0) ALIAS(distributed_directory_monitor_sleep_time_ms) \ - M(Milliseconds, distributed_background_insert_max_sleep_time_ms, 30000, "Maximum sleep time for background INSERTs into Distributed, it limits exponential growth too.", 0) ALIAS(distributed_directory_monitor_max_sleep_time_ms) \ - \ - M(Bool, distributed_background_insert_batch, false, "Should background INSERTs into Distributed be batched into bigger blocks.", 0) ALIAS(distributed_directory_monitor_batch_inserts) \ - M(Bool, distributed_background_insert_split_batch_on_failure, false, "Should batches of the background INSERT into Distributed be split into smaller batches in case of failures.", 0) ALIAS(distributed_directory_monitor_split_batch_on_failure) \ - \ - M(Bool, optimize_move_to_prewhere, true, "Allows disabling WHERE to PREWHERE optimization in SELECT queries from MergeTree.", 0) \ - M(Bool, optimize_move_to_prewhere_if_final, false, "If query has `FINAL`, the optimization `move_to_prewhere` is not always correct and it is enabled only if both settings `optimize_move_to_prewhere` and `optimize_move_to_prewhere_if_final` are turned on", 0) \ - M(Bool, move_all_conditions_to_prewhere, true, "Move all viable conditions from WHERE to PREWHERE", 0) \ - M(Bool, enable_multiple_prewhere_read_steps, true, "Move more conditions from WHERE to PREWHERE and do reads from disk and filtering in multiple steps if there are multiple conditions combined with AND", 0) \ - M(Bool, move_primary_key_columns_to_end_of_prewhere, true, "Move PREWHERE conditions containing primary key columns to the end of AND chain. It is likely that these conditions are taken into account during primary key analysis and thus will not contribute a lot to PREWHERE filtering.", 0) \ - \ - M(UInt64, alter_sync, 1, "Wait for actions to manipulate the partitions. 0 - do not wait, 1 - wait for execution only of itself, 2 - wait for everyone.", 0) ALIAS(replication_alter_partitions_sync) \ - M(Int64, replication_wait_for_inactive_replica_timeout, 120, "Wait for inactive replica to execute ALTER/OPTIMIZE. Time in seconds, 0 - do not wait, negative - wait for unlimited time.", 0) \ - M(Bool, alter_move_to_space_execute_async, false, "Execute ALTER TABLE MOVE ... TO [DISK|VOLUME] asynchronously", 0) \ - \ - M(LoadBalancing, load_balancing, LoadBalancing::RANDOM, "Which replicas (among healthy replicas) to preferably send a query to (on the first attempt) for distributed processing.", 0) \ - M(UInt64, load_balancing_first_offset, 0, "Which replica to preferably send a query when FIRST_OR_RANDOM load balancing strategy is used.", 0) \ - \ - M(TotalsMode, totals_mode, TotalsMode::AFTER_HAVING_EXCLUSIVE, "How to calculate TOTALS when HAVING is present, as well as when max_rows_to_group_by and group_by_overflow_mode = ‘any’ are present.", IMPORTANT) \ - M(Float, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.", 0) \ - \ - M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \ - M(Bool, allow_suspicious_fixed_string_types, false, "In CREATE TABLE statement allows creating columns of type FixedString(n) with n > 256. FixedString with length >= 256 is suspicious and most likely indicates misuse", 0) \ - M(Bool, allow_suspicious_indices, false, "Reject primary/secondary indexes and sorting keys with identical expressions", 0) \ - M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \ - M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \ - M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \ - M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code. Due to a bug in the LLVM compiler infrastructure, on AArch64 machines, it is known to lead to a nullptr dereference and, consequently, server crash. Do not enable this setting.", 0) \ - M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ - M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ - M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ - M(Bool, compile_sort_description, true, "Compile sort description to native code.", 0) \ - M(UInt64, min_count_to_compile_sort_description, 3, "The number of identical sort descriptions before they are JIT-compiled", 0) \ - M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ - M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ - M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ - M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ - M(Bool, enable_memory_bound_merging_of_aggregation_results, true, "Enable memory bound merging strategy for aggregation.", 0) \ - M(Bool, enable_positional_arguments, true, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ - M(Bool, enable_extended_results_for_datetime_functions, false, "Enable date functions like toLastDayOfMonth return Date32 results (instead of Date results) for Date32/DateTime64 arguments.", 0) \ - M(Bool, allow_nonconst_timezone_arguments, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()", 0) \ - M(Bool, function_locate_has_mysql_compatible_argument_order, true, "Function locate() has arguments (needle, haystack[, start_pos]) like in MySQL instead of (haystack, needle[, start_pos]) like function position()", 0) \ - \ - M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ - \ - M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \ - \ - M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards. Shard is marked as unavailable when: 1) The shard cannot be reached due to a connection failure. 2) Shard is unresolvable through DNS. 3) Table does not exist on the shard.", 0) \ - \ - M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard; if set to 1 - SELECT is executed on each shard; if set to 2 - SELECT and INSERT are executed on each shard", 0) \ - M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ - M(UInt64, distributed_push_down_limit, 1, "If 1, LIMIT will be applied on each shard separately. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.", 0) \ - M(Bool, optimize_distributed_group_by_sharding_key, true, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ - M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ - M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ - M(Bool, optimize_skip_unused_shards_rewrite_in, true, "Rewrite IN in query for remote shards to exclude values that does not belong to the shard (requires optimize_skip_unused_shards)", 0) \ - M(Bool, allow_nondeterministic_optimize_skip_unused_shards, false, "Allow non-deterministic functions (includes dictGet) in sharding_key for optimize_skip_unused_shards", 0) \ - M(UInt64, force_optimize_skip_unused_shards, 0, "Throw an exception if unused shards cannot be skipped (1 - throw only if the table has the sharding key, 2 - always throw.", 0) \ - M(UInt64, optimize_skip_unused_shards_nesting, 0, "Same as optimize_skip_unused_shards, but accept nesting level until which it will work.", 0) \ - M(UInt64, force_optimize_skip_unused_shards_nesting, 0, "Same as force_optimize_skip_unused_shards, but accept nesting level until which it will work.", 0) \ - \ - M(Bool, input_format_parallel_parsing, true, "Enable parallel parsing for some data formats.", 0) \ - M(UInt64, min_chunk_bytes_for_parallel_parsing, (10 * 1024 * 1024), "The minimum chunk size in bytes, which each thread will parse in parallel.", 0) \ - M(Bool, output_format_parallel_formatting, true, "Enable parallel formatting for some data formats.", 0) \ - M(UInt64, output_format_compression_level, 3, "Default compression level if query output is compressed. The setting is applied when `SELECT` query has `INTO OUTFILE` or when inserting to table function `file`, `url`, `hdfs`, `s3`, and `azureBlobStorage`.", 0) \ - M(UInt64, output_format_compression_zstd_window_log, 0, "Can be used when the output compression method is `zstd`. If greater than `0`, this setting explicitly sets compression window size (power of `2`) and enables a long-range mode for zstd compression.", 0) \ - \ - M(UInt64, merge_tree_min_rows_for_concurrent_read, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized.", 0) \ - M(UInt64, merge_tree_min_bytes_for_concurrent_read, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized.", 0) \ - M(UInt64, merge_tree_min_rows_for_seek, 0, "You can skip reading more than that number of rows at the price of one seek per file.", 0) \ - M(UInt64, merge_tree_min_bytes_for_seek, 0, "You can skip reading more than that number of bytes at the price of one seek per file.", 0) \ - M(UInt64, merge_tree_coarse_index_granularity, 8, "If the index segment can contain the required keys, divide it into as many parts and recursively check them.", 0) \ - M(UInt64, merge_tree_max_rows_to_use_cache, (128 * 8192), "The maximum number of rows per request, to use the cache of uncompressed data. If the request is large, the cache is not used. (For large queries not to flush out the cache.)", 0) \ - M(UInt64, merge_tree_max_bytes_to_use_cache, (192 * 10 * 1024 * 1024), "The maximum number of bytes per request, to use the cache of uncompressed data. If the request is large, the cache is not used. (For large queries not to flush out the cache.)", 0) \ - M(Bool, do_not_merge_across_partitions_select_final, false, "Merge parts only in one partition in select final", 0) \ - M(Bool, split_parts_ranges_into_intersecting_and_non_intersecting_final, true, "Split parts ranges into intersecting and non intersecting during FINAL optimization", 0) \ - M(Bool, split_intersecting_parts_ranges_into_layers_final, true, "Split intersecting parts ranges into layers during FINAL optimization", 0) \ - \ - M(UInt64, mysql_max_rows_to_insert, 65536, "The maximum number of rows in MySQL batch insertion of the MySQL storage engine", 0) \ - M(Bool, mysql_map_string_to_text_in_show_columns, true, "If enabled, String type will be mapped to TEXT in SHOW [FULL] COLUMNS, BLOB otherwise. Has an effect only when the connection is made through the MySQL wire protocol.", 0) \ - M(Bool, mysql_map_fixed_string_to_text_in_show_columns, true, "If enabled, FixedString type will be mapped to TEXT in SHOW [FULL] COLUMNS, BLOB otherwise. Has an effect only when the connection is made through the MySQL wire protocol.", 0) \ - \ - M(UInt64, optimize_min_equality_disjunction_chain_length, 3, "The minimum length of the expression `expr = x1 OR ... expr = xN` for optimization ", 0) \ - M(UInt64, optimize_min_inequality_conjunction_chain_length, 3, "The minimum length of the expression `expr <> x1 AND ... expr <> xN` for optimization ", 0) \ - \ - M(UInt64, min_bytes_to_use_direct_io, 0, "The minimum number of bytes for reading the data with O_DIRECT option during SELECT queries execution. 0 - disabled.", 0) \ - M(UInt64, min_bytes_to_use_mmap_io, 0, "The minimum number of bytes for reading the data with mmap option during SELECT queries execution. 0 - disabled.", 0) \ - M(Bool, checksum_on_read, true, "Validate checksums on reading. It is enabled by default and should be always enabled in production. Please do not expect any benefits in disabling this setting. It may only be used for experiments and benchmarks. The setting only applicable for tables of MergeTree family. Checksums are always validated for other table engines and when receiving data over network.", 0) \ - \ - M(Bool, force_index_by_date, false, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \ - M(Bool, force_primary_key, false, "Throw an exception if there is primary key in a table, and it is not used.", 0) \ - M(Bool, use_skip_indexes, true, "Use data skipping indexes during query execution.", 0) \ - M(Bool, use_skip_indexes_if_final, false, "If query has FINAL, then skipping data based on indexes may produce incorrect result, hence disabled by default.", 0) \ - M(Bool, materialize_skip_indexes_on_insert, true, "If true skip indexes are calculated on inserts, otherwise skip indexes will be calculated only during merges", 0) \ - M(Bool, materialize_statistics_on_insert, true, "If true statistics are calculated on inserts, otherwise statistics will be calculated only during merges", 0) \ - M(String, ignore_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be excluded during query execution.", 0) \ - \ - M(String, force_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be used during query execution, otherwise an exception will be thrown.", 0) \ - \ - M(Float, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \ - M(Float, max_streams_multiplier_for_merge_tables, 5, "Ask more streams when reading from Merge table. Streams will be spread across tables that Merge table will use. This allows more even distribution of work across threads and especially helpful when merged tables differ in size.", 0) \ - \ - M(String, network_compression_method, "LZ4", "Allows you to select the method of data compression when writing.", 0) \ - \ - M(Int64, network_zstd_compression_level, 1, "Allows you to select the level of ZSTD compression.", 0) \ - \ - M(Int64, zstd_window_log_max, 0, "Allows you to select the max window log of ZSTD (it will not be used for MergeTree family)", 0) \ - \ - M(UInt64, priority, 0, "Priority of the query. 1 - the highest, higher value - lower priority; 0 - do not use priorities.", 0) \ - M(Int64, os_thread_priority, 0, "If non zero - set corresponding 'nice' value for query processing threads. Can be used to adjust query priority for OS scheduler.", 0) \ - \ - M(Bool, log_queries, true, "Log requests and write the log to the system table.", 0) \ - M(Bool, log_formatted_queries, false, "Log formatted queries and write the log to the system table.", 0) \ - M(LogQueriesType, log_queries_min_type, QueryLogElementType::QUERY_START, "Minimal type in query_log to log, possible values (from low to high): QUERY_START, QUERY_FINISH, EXCEPTION_BEFORE_START, EXCEPTION_WHILE_PROCESSING.", 0) \ - M(Milliseconds, log_queries_min_query_duration_ms, 0, "Minimal time for the query to run, to get to the query_log/query_thread_log/query_views_log.", 0) \ - M(UInt64, log_queries_cut_to_length, 100000, "If query length is greater than specified threshold (in bytes), then cut query when writing to query log. Also limit length of printed query in ordinary text log.", 0) \ - M(Float, log_queries_probability, 1., "Log queries with the specified probability.", 0) \ - \ - M(Bool, log_processors_profiles, true, "Log Processors profile events.", 0) \ - M(DistributedProductMode, distributed_product_mode, DistributedProductMode::DENY, "How are distributed subqueries performed inside IN or JOIN sections?", IMPORTANT) \ - \ - M(UInt64, max_concurrent_queries_for_all_users, 0, "The maximum number of concurrent requests for all users.", 0) \ - M(UInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.", 0) \ - \ - M(Bool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of inserting blocks should be performed", 0) \ - M(Bool, async_insert_deduplicate, false, "For async INSERT queries in the replicated table, specifies that deduplication of inserting blocks should be performed", 0) \ - \ - M(UInt64Auto, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled, 'auto' - use majority", 0) \ - M(Milliseconds, insert_quorum_timeout, 600000, "If the quorum of replicas did not meet in specified time (in milliseconds), exception will be thrown and insertion is aborted.", 0) \ - M(Bool, insert_quorum_parallel, true, "For quorum INSERT queries - enable to make parallel inserts without linearizability", 0) \ - M(UInt64, select_sequential_consistency, 0, "For SELECT queries from the replicated table, throw an exception if the replica does not have a chunk written with the quorum; do not read the parts that have not yet been written with the quorum.", 0) \ - M(UInt64, table_function_remote_max_addresses, 1000, "The maximum number of different shards and the maximum number of replicas of one shard in the `remote` function.", 0) \ - M(Milliseconds, read_backoff_min_latency_ms, 1000, "Setting to reduce the number of threads in case of slow reads. Pay attention only to reads that took at least that much time.", 0) \ - M(UInt64, read_backoff_max_throughput, 1048576, "Settings to reduce the number of threads in case of slow reads. Count events when the read bandwidth is less than that many bytes per second.", 0) \ - M(Milliseconds, read_backoff_min_interval_between_events_ms, 1000, "Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time.", 0) \ - M(UInt64, read_backoff_min_events, 2, "Settings to reduce the number of threads in case of slow reads. The number of events after which the number of threads will be reduced.", 0) \ - \ - M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ - \ - M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ - M(Float, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability.", 0) \ - \ - M(Bool, enable_http_compression, false, "Compress the result if the client over HTTP said that it understands data compressed by gzip, deflate, zstd, br, lz4, bz2, xz.", 0) \ - M(Int64, http_zlib_compression_level, 3, "Compression level - used if the client on HTTP said that it understands data compressed by gzip or deflate.", 0) \ - \ - M(Bool, http_native_compression_disable_checksumming_on_decompress, false, "If you uncompress the POST data from the client compressed by the native format, do not check the checksum.", 0) \ - \ - M(String, count_distinct_implementation, "uniqExact", "What aggregate function to use for implementation of count(DISTINCT ...)", 0) \ - \ - M(Bool, add_http_cors_header, false, "Write add http CORS header.", 0) \ - \ - M(UInt64, max_http_get_redirects, 0, "Max number of http GET redirects hops allowed. Ensures additional security measures are in place to prevent a malicious server to redirect your requests to unexpected services.\n\nIt is the case when an external server redirects to another address, but that address appears to be internal to the company's infrastructure, and by sending an HTTP request to an internal server, you could request an internal API from the internal network, bypassing the auth, or even query other services, such as Redis or Memcached. When you don't have an internal infrastructure (including something running on your localhost), or you trust the server, it is safe to allow redirects. Although keep in mind, that if the URL uses HTTP instead of HTTPS, and you will have to trust not only the remote server but also your ISP and every network in the middle.", 0) \ - \ - M(Bool, use_client_time_zone, false, "Use client timezone for interpreting DateTime string values, instead of adopting server timezone.", 0) \ - \ - M(Bool, send_progress_in_http_headers, false, "Send progress notifications using X-ClickHouse-Progress headers. Some clients do not support high amount of HTTP headers (Python requests in particular), so it is disabled by default.", 0) \ - \ - M(UInt64, http_headers_progress_interval_ms, 100, "Do not send HTTP headers X-ClickHouse-Progress more frequently than at each specified interval.", 0) \ - M(Bool, http_wait_end_of_query, false, "Enable HTTP response buffering on the server-side.", 0) \ - M(Bool, http_write_exception_in_output_format, true, "Write exception in output format to produce valid output. Works with JSON and XML formats.", 0) \ - M(UInt64, http_response_buffer_size, 0, "The number of bytes to buffer in the server memory before sending a HTTP response to the client or flushing to disk (when http_wait_end_of_query is enabled).", 0) \ - \ - M(Bool, fsync_metadata, true, "Do fsync after changing metadata for tables and databases (.sql files). Could be disabled in case of poor latency on server with high load of DDL queries and high load of disk subsystem.", 0) \ - \ - M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ - \ - M(Int32, join_output_by_rowlist_perkey_rows_threshold, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join.", 0) \ - M(JoinStrictness, join_default_strictness, JoinStrictness::All, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ - M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ - M(Bool, single_join_prefer_left_table, true, "For single JOIN in case of identifier ambiguity prefer left table", IMPORTANT) \ - \ - M(UInt64, preferred_block_size_bytes, 1000000, "This setting adjusts the data block size for query processing and represents additional fine tune to the more rough 'max_block_size' setting. If the columns are large and with 'max_block_size' rows the block size is likely to be larger than the specified amount of bytes, its size will be lowered for better CPU cache locality.", 0) \ - \ - M(UInt64, max_replica_delay_for_distributed_queries, 300, "If set, distributed queries of Replicated tables will choose servers with replication delay in seconds less than the specified value (not inclusive). Zero means do not take delay into account.", 0) \ - M(Bool, fallback_to_stale_replicas_for_distributed_queries, true, "Suppose max_replica_delay_for_distributed_queries is set and all replicas for the queried table are stale. If this setting is enabled, the query will be performed anyway, otherwise the error will be reported.", 0) \ - M(UInt64, preferred_max_column_in_block_size_bytes, 0, "Limit on max column size in block while reading. Helps to decrease cache misses count. Should be close to L2 cache size.", 0) \ - \ - M(UInt64, parts_to_delay_insert, 0, "If the destination table contains at least that many active parts in a single partition, artificially slow down insert into table.", 0) \ - M(UInt64, parts_to_throw_insert, 0, "If more than this number active parts in a single partition of the destination table, throw 'Too many parts ...' exception.", 0) \ - M(UInt64, number_of_mutations_to_delay, 0, "If the mutated table contains at least that many unfinished mutations, artificially slow down mutations of table. 0 - disabled", 0) \ - M(UInt64, number_of_mutations_to_throw, 0, "If the mutated table contains at least that many unfinished mutations, throw 'Too many mutations ...' exception. 0 - disabled", 0) \ - M(Int64, distributed_ddl_task_timeout, 180, "Timeout for DDL query responses from all hosts in cluster. If a ddl request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. Negative value means infinite. Zero means async mode.", 0) \ - M(Milliseconds, stream_flush_interval_ms, 7500, "Timeout for flushing data from streaming storages.", 0) \ - M(Milliseconds, stream_poll_timeout_ms, 500, "Timeout for polling data from/to streaming storages.", 0) \ - \ - M(Bool, final, false, "Query with the FINAL modifier by default. If the engine does not support final, it does not have any effect. On queries with multiple tables final is applied only on those that support it. It also works on distributed tables", 0) \ - \ - M(Bool, partial_result_on_first_cancel, false, "Allows query to return a partial result after cancel.", 0) \ - \ - M(Bool, ignore_on_cluster_for_replicated_udf_queries, false, "Ignore ON CLUSTER clause for replicated UDF management queries.", 0) \ - M(Bool, ignore_on_cluster_for_replicated_access_entities_queries, false, "Ignore ON CLUSTER clause for replicated access entities management queries.", 0) \ - M(Bool, ignore_on_cluster_for_replicated_named_collections_queries, false, "Ignore ON CLUSTER clause for replicated named collections management queries.", 0) \ - /** Settings for testing hedged requests */ \ - M(Milliseconds, sleep_in_send_tables_status_ms, 0, "Time to sleep in sending tables status response in TCPHandler", 0) \ - M(Milliseconds, sleep_in_send_data_ms, 0, "Time to sleep in sending data in TCPHandler", 0) \ - M(Milliseconds, sleep_after_receiving_query_ms, 0, "Time to sleep after receiving query in TCPHandler", 0) \ - M(UInt64, unknown_packet_in_send_data, 0, "Send unknown packet instead of data Nth data packet", 0) \ - \ - M(Bool, insert_allow_materialized_columns, false, "If setting is enabled, Allow materialized columns in INSERT.", 0) \ - M(Seconds, http_connection_timeout, DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT, "HTTP connection timeout.", 0) \ - M(Seconds, http_send_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP send timeout", 0) \ - M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \ - M(UInt64, http_max_uri_size, 1048576, "Maximum URI length of HTTP request", 0) \ - M(UInt64, http_max_fields, 1000000, "Maximum number of fields in HTTP header", 0) \ - M(UInt64, http_max_field_name_size, 128 * 1024, "Maximum length of field name in HTTP header", 0) \ - M(UInt64, http_max_field_value_size, 128 * 1024, "Maximum length of field value in HTTP header", 0) \ - M(Bool, http_skip_not_found_url_for_globs, true, "Skip URLs for globs with HTTP_NOT_FOUND error", 0) \ - M(Bool, http_make_head_request, true, "Allows the execution of a `HEAD` request while reading data from HTTP to retrieve information about the file to be read, such as its size", 0) \ - M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \ - M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \ - M(UInt64, use_index_for_in_with_subqueries_max_values, 0, "The maximum size of set in the right hand side of the IN operator to use table index for filtering. It allows to avoid performance degradation and higher memory usage due to preparation of additional data structures for large queries. Zero means no limit.", 0) \ - M(Bool, analyze_index_with_space_filling_curves, true, "If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.", 0) \ - M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \ - M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ - M(Bool, empty_result_for_aggregation_by_constant_keys_on_empty_set, true, "Return empty result when aggregating by constant keys on empty set.", 0) \ - M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ - M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ - M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \ - M(Bool, enable_zstd_qat_codec, false, "Enable/disable the ZSTD_QAT codec.", 0) \ - M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ - M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ - M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ - M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \ - M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ - M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ - M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - \ - M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ - M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \ - \ - \ - /** Limits during query execution are part of the settings. \ - * Used to provide a more safe execution of queries from the user interface. \ - * Basically, limits are checked for each block (not every row). That is, the limits can be slightly violated. \ - * Almost all limits apply only to SELECTs. \ - * Almost all limits apply to each stream individually. \ - */ \ - \ - M(UInt64, max_rows_to_read, 0, "Limit on read rows from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \ - M(UInt64, max_bytes_to_read, 0, "Limit on read bytes (after decompression) from the most 'deep' sources. That is, only in the deepest subquery. When reading from a remote server, it is only checked on a remote server.", 0) \ - M(OverflowMode, read_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - \ - M(UInt64, max_rows_to_read_leaf, 0, "Limit on read rows on the leaf nodes for distributed queries. Limit is applied for local reads only, excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \ - M(UInt64, max_bytes_to_read_leaf, 0, "Limit on read bytes (after decompression) on the leaf nodes for distributed queries. Limit is applied for local reads only, excluding the final merge stage on the root node. Note, the setting is unstable with prefer_localhost_replica=1.", 0) \ - M(OverflowMode, read_overflow_mode_leaf, OverflowMode::THROW, "What to do when the leaf limit is exceeded.", 0) \ - \ - M(UInt64, max_rows_to_group_by, 0, "If aggregation during GROUP BY is generating more than the specified number of rows (unique GROUP BY keys), the behavior will be determined by the 'group_by_overflow_mode' which by default is - throw an exception, but can be also switched to an approximate GROUP BY mode.", 0) \ - M(OverflowModeGroupBy, group_by_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - M(UInt64, max_bytes_before_external_group_by, 0, "If memory usage during GROUP BY operation is exceeding this threshold in bytes, activate the 'external aggregation' mode (spill data to disk). Recommended value is half of available system memory.", 0) \ - \ - M(UInt64, max_rows_to_sort, 0, "If more than the specified amount of records have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \ - M(UInt64, max_bytes_to_sort, 0, "If more than the specified amount of (uncompressed) bytes have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \ - M(OverflowMode, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - M(UInt64, prefer_external_sort_block_bytes, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging.", 0) \ - M(UInt64, max_bytes_before_external_sort, 0, "If memory usage during ORDER BY operation is exceeding this threshold in bytes, activate the 'external sorting' mode (spill data to disk). Recommended value is half of available system memory.", 0) \ - M(UInt64, max_bytes_before_remerge_sort, 1000000000, "In case of ORDER BY with LIMIT, when memory usage is higher than specified threshold, perform additional steps of merging blocks before final merge to keep just top LIMIT rows.", 0) \ - M(Float, remerge_sort_lowered_memory_bytes_ratio, 2., "If memory usage after remerge does not reduced by this ratio, remerge will be disabled.", 0) \ - \ - M(UInt64, max_result_rows, 0, "Limit on result size in rows. The query will stop after processing a block of data if the threshold is met, but it will not cut the last block of the result, therefore the result size can be larger than the threshold.", 0) \ - M(UInt64, max_result_bytes, 0, "Limit on result size in bytes (uncompressed). The query will stop after processing a block of data if the threshold is met, but it will not cut the last block of the result, therefore the result size can be larger than the threshold. Caveats: the result size in memory is taken into account for this threshold. Even if the result size is small, it can reference larger data structures in memory, representing dictionaries of LowCardinality columns, and Arenas of AggregateFunction columns, so the threshold can be exceeded despite the small result size. The setting is fairly low level and should be used with caution.", 0) \ - M(OverflowMode, result_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - \ - /* TODO: Check also when merging and finalizing aggregate functions. */ \ - M(Seconds, max_execution_time, 0, "If query runtime exceeds the specified number of seconds, the behavior will be determined by the 'timeout_overflow_mode', which by default is - throw an exception. Note that the timeout is checked and query can stop only in designated places during data processing. It currently cannot stop during merging of aggregation states or during query analysis, and the actual run time will be higher than the value of this setting.", 0) \ - M(OverflowMode, timeout_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - M(Seconds, max_execution_time_leaf, 0, "Similar semantic to max_execution_time but only apply on leaf node for distributed queries, the time out behavior will be determined by 'timeout_overflow_mode_leaf' which by default is - throw an exception", 0) \ - M(OverflowMode, timeout_overflow_mode_leaf, OverflowMode::THROW, "What to do when the leaf limit is exceeded.", 0) \ - \ - M(UInt64, min_execution_speed, 0, "Minimum number of execution rows per second.", 0) \ - M(UInt64, max_execution_speed, 0, "Maximum number of execution rows per second.", 0) \ - M(UInt64, min_execution_speed_bytes, 0, "Minimum number of execution bytes per second.", 0) \ - M(UInt64, max_execution_speed_bytes, 0, "Maximum number of execution bytes per second.", 0) \ - M(Seconds, timeout_before_checking_execution_speed, 10, "Check that the speed is not too low after the specified time has elapsed.", 0) \ - M(Seconds, max_estimated_execution_time, 0, "Maximum query estimate execution time in seconds.", 0) \ - \ - M(UInt64, max_columns_to_read, 0, "If a query requires reading more than specified number of columns, exception is thrown. Zero value means unlimited. This setting is useful to prevent too complex queries.", 0) \ - M(UInt64, max_temporary_columns, 0, "If a query generates more than the specified number of temporary columns in memory as a result of intermediate calculation, exception is thrown. Zero value means unlimited. This setting is useful to prevent too complex queries.", 0) \ - M(UInt64, max_temporary_non_const_columns, 0, "Similar to the 'max_temporary_columns' setting but applies only to non-constant columns. This makes sense, because constant columns are cheap and it is reasonable to allow more of them.", 0) \ - \ - M(UInt64, max_sessions_for_user, 0, "Maximum number of simultaneous sessions for a user.", 0) \ - \ - M(UInt64, max_subquery_depth, 100, "If a query has more than the specified number of nested subqueries, throw an exception. This allows you to have a sanity check to protect the users of your cluster from going insane with their queries.", 0) \ - M(UInt64, max_analyze_depth, 5000, "Maximum number of analyses performed by interpreter.", 0) \ - M(UInt64, max_ast_depth, 1000, "Maximum depth of query syntax tree. Checked after parsing.", 0) \ - M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \ - M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \ - \ - M(UInt64, readonly, 0, "0 - no read-only restrictions. 1 - only read requests, as well as changing explicitly allowed settings. 2 - only read requests, as well as changing settings, except for the 'readonly' setting.", 0) \ - \ - M(UInt64, max_rows_in_set, 0, "Maximum size of the set (in number of elements) resulting from the execution of the IN section.", 0) \ - M(UInt64, max_bytes_in_set, 0, "Maximum size of the set (in bytes in memory) resulting from the execution of the IN section.", 0) \ - M(OverflowMode, set_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - \ - M(UInt64, max_rows_in_join, 0, "Maximum size of the hash table for JOIN (in number of rows).", 0) \ - M(UInt64, max_bytes_in_join, 0, "Maximum size of the hash table for JOIN (in number of bytes in memory).", 0) \ - M(OverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - M(Bool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key. Can be applied only to hash join and storage join.", IMPORTANT) \ - M(JoinAlgorithm, join_algorithm, JoinAlgorithm::DEFAULT, "Specify join algorithm.", 0) \ - M(UInt64, cross_join_min_rows_to_compress, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.", 0) \ - M(UInt64, cross_join_min_bytes_to_compress, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.", 0) \ - M(UInt64, default_max_bytes_in_join, 1000000000, "Maximum size of right-side table if limit is required but max_bytes_in_join is not set.", 0) \ - M(UInt64, partial_merge_join_left_table_buffer_bytes, 0, "If not 0 group left table blocks in bigger ones for left-side table in partial merge join. It uses up to 2x of specified memory per joining thread.", 0) \ - M(UInt64, partial_merge_join_rows_in_right_blocks, 65536, "Split right-hand joining data in blocks of specified size. It's a portion of data indexed by min-max values and possibly unloaded on disk.", 0) \ - M(UInt64, join_on_disk_max_files_to_merge, 64, "For MergeJoin on disk set how much files it's allowed to sort simultaneously. Then this value bigger then more memory used and then less disk I/O needed. Minimum is 2.", 0) \ - M(UInt64, max_rows_in_set_to_optimize_join, 0, "Maximal size of the set to filter joined tables by each other row sets before joining. 0 - disable.", 0) \ - \ - M(Bool, compatibility_ignore_collation_in_create_table, true, "Compatibility ignore collation in create table", 0) \ - \ - M(String, temporary_files_codec, "LZ4", "Set compression codec for temporary files produced by (JOINs, external GROUP BY, external ORDER BY). I.e. LZ4, NONE.", 0) \ - \ - M(UInt64, max_rows_to_transfer, 0, "Maximum size (in rows) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ - M(UInt64, max_bytes_to_transfer, 0, "Maximum size (in uncompressed bytes) of the transmitted external table obtained when the GLOBAL IN/JOIN section is executed.", 0) \ - M(OverflowMode, transfer_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - \ - M(UInt64, max_rows_in_distinct, 0, "Maximum number of elements during execution of DISTINCT.", 0) \ - M(UInt64, max_bytes_in_distinct, 0, "Maximum total size of state (in uncompressed bytes) in memory for the execution of DISTINCT.", 0) \ - M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - \ - M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ - M(UInt64, memory_overcommit_ratio_denominator, 1_GiB, "It represents soft memory limit on the user level. This value is used to compute query overcommit ratio.", 0) \ - M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ - M(UInt64, memory_overcommit_ratio_denominator_for_user, 1_GiB, "It represents soft memory limit on the global level. This value is used to compute query overcommit ratio.", 0) \ - M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \ - M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ - M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation (can be changed with `memory_profiler_sample_min_allocation_size` and `memory_profiler_sample_max_allocation_size`). Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ - M(UInt64, memory_profiler_sample_min_allocation_size, 0, "Collect random allocations of size greater or equal than specified value with probability equal to `memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \ - M(UInt64, memory_profiler_sample_max_allocation_size, 0, "Collect random allocations of size less or equal than specified value with probability equal to `memory_profiler_sample_probability`. 0 means disabled. You may want to set 'max_untracked_memory' to 0 to make this threshold to work as expected.", 0) \ - M(Bool, trace_profile_events, false, "Send to system.trace_log profile event and value of increment on each increment with 'ProfileEvent' trace_type", 0) \ - \ - M(UInt64, memory_usage_overcommit_max_wait_microseconds, 5'000'000, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown.", 0) \ - \ - M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ - M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ - M(UInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.", 0)\ - M(UInt64, max_network_bandwidth_for_all_users, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running queries. Zero means unlimited.", 0) \ - \ - M(UInt64, max_temporary_data_on_disk_size_for_user, 0, "The maximum amount of data consumed by temporary files on disk in bytes for all concurrently running user queries. Zero means unlimited.", 0)\ - M(UInt64, max_temporary_data_on_disk_size_for_query, 0, "The maximum amount of data consumed by temporary files on disk in bytes for all concurrently running queries. Zero means unlimited.", 0)\ - \ - M(UInt64, backup_restore_keeper_max_retries, 20, "Max retries for keeper operations during backup or restore", 0) \ - M(UInt64, backup_restore_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for [Zoo]Keeper operations during backup or restore", 0) \ - M(UInt64, backup_restore_keeper_retry_max_backoff_ms, 5000, "Max backoff timeout for [Zoo]Keeper operations during backup or restore", 0) \ - M(Float, backup_restore_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during backup or restore. Valid value is in interval [0.0f, 1.0f]", 0) \ - M(UInt64, backup_restore_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ - M(UInt64, backup_restore_keeper_value_max_size, 1048576, "Maximum size of data of a [Zoo]Keeper's node during backup", 0) \ - M(UInt64, backup_restore_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup or restore", 0) \ - M(UInt64, backup_restore_batch_size_for_keeper_multi, 1000, "Maximum size of batch for multi request to [Zoo]Keeper during backup or restore", 0) \ - M(UInt64, backup_restore_s3_retry_attempts, 1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore.", 0) \ - M(UInt64, max_backup_bandwidth, 0, "The maximum read speed in bytes per second for particular backup on server. Zero means unlimited.", 0) \ - \ - M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ - M(Bool, log_query_settings, true, "Log query settings into the query_log.", 0) \ - M(Bool, log_query_threads, false, "Log query threads into system.query_thread_log table. This setting have effect only when 'log_queries' is true.", 0) \ - M(Bool, log_query_views, true, "Log query dependent views into system.query_views_log table. This setting have effect only when 'log_queries' is true.", 0) \ - M(String, log_comment, "", "Log comment into system.query_log table and server log. It can be set to arbitrary string no longer than max_query_size.", 0) \ - M(LogsLevel, send_logs_level, LogsLevel::fatal, "Send server text logs with specified minimum level to client. Valid values: 'trace', 'debug', 'information', 'warning', 'error', 'fatal', 'none'", 0) \ - M(String, send_logs_source_regexp, "", "Send server text logs with specified regexp to match log source name. Empty means all sources.", 0) \ - M(Bool, enable_optimize_predicate_expression, true, "If it is set to true, optimize predicates to subqueries.", 0) \ - M(Bool, enable_optimize_predicate_expression_to_final_subquery, true, "Allow push predicate to final subquery.", 0) \ - M(Bool, allow_push_predicate_when_subquery_contains_with, true, "Allows push predicate when subquery contains WITH clause", 0) \ - \ - M(UInt64, low_cardinality_max_dictionary_size, 8192, "Maximum size (in rows) of shared global dictionary for LowCardinality type.", 0) \ - M(Bool, low_cardinality_use_single_dictionary_for_part, false, "LowCardinality type serialization setting. If is true, than will use additional keys when global dictionary overflows. Otherwise, will create several shared dictionaries.", 0) \ - M(Bool, decimal_check_overflow, true, "Check overflow of decimal arithmetic/comparison operations", 0) \ - M(Bool, allow_custom_error_code_in_throwif, false, "Enable custom error code in function throwIf(). If true, thrown exceptions may have unexpected error codes.", 0) \ - \ - M(Bool, prefer_localhost_replica, true, "If it's true then queries will be always sent to local replica (if it exists). If it's false then replica to send a query will be chosen between local and remote ones according to load_balancing", 0) \ - M(UInt64, max_fetch_partition_retries_count, 5, "Amount of retries while fetching partition from another host.", 0) \ - M(UInt64, http_max_multipart_form_data_size, 1024 * 1024 * 1024, "Limit on size of multipart/form-data content. This setting cannot be parsed from URL parameters and should be set in user profile. Note that content is parsed and external tables are created in memory before start of query execution. And this is the only limit that has effect on that stage (limits on max memory usage and max execution time have no effect while reading HTTP form data).", 0) \ - M(Bool, calculate_text_stack_trace, true, "Calculate text stack trace in case of exceptions during query execution. This is the default. It requires symbol lookups that may slow down fuzzing tests when huge amount of wrong queries are executed. In normal cases you should not disable this option.", 0) \ - M(Bool, enable_job_stack_trace, false, "Output stack trace of a job creator when job results in exception", 0) \ - M(Bool, allow_ddl, true, "If it is set to true, then a user is allowed to executed DDL queries.", 0) \ - M(Bool, parallel_view_processing, false, "Enables pushing to attached views concurrently instead of sequentially.", 0) \ - M(Bool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \ - M(Bool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \ - M(Bool, optimize_read_in_window_order, true, "Enable ORDER BY optimization in window clause for reading data in corresponding order in MergeTree tables.", 0) \ - M(Bool, optimize_aggregation_in_order, false, "Enable GROUP BY optimization for aggregating data in corresponding order in MergeTree tables.", 0) \ - M(Bool, read_in_order_use_buffering, true, "Use buffering before merging while reading in order of primary key. It increases the parallelism of query execution", 0) \ - M(UInt64, aggregation_in_order_max_block_bytes, 50000000, "Maximal size of block in bytes accumulated during aggregation in order of primary key. Lower block size allows to parallelize more final merge stage of aggregation.", 0) \ - M(UInt64, read_in_order_two_level_merge_threshold, 100, "Minimal number of parts to read to run preliminary merge step during multithread reading in order of primary key.", 0) \ - M(Bool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \ - M(Bool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ - M(Bool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql', 'postgresql' and 'odbc' table functions.", 0) \ - M(Bool, external_table_strict_query, false, "If it is set to true, transforming expression to local filter is forbidden for queries to external tables.", 0) \ - \ - M(Bool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ - M(UInt64, max_hyperscan_regexp_length, 0, "Max length of regexp than can be used in hyperscan multi-match functions. Zero means unlimited.", 0) \ - M(UInt64, max_hyperscan_regexp_total_length, 0, "Max total length of all regexps than can be used in hyperscan multi-match functions (per every function). Zero means unlimited.", 0) \ - M(Bool, reject_expensive_hyperscan_regexps, true, "Reject patterns which will likely be expensive to evaluate with hyperscan (due to NFA state explosion)", 0) \ - M(Bool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ - M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ - M(Bool, splitby_max_substrings_includes_remaining_string, false, "Functions 'splitBy*()' with 'max_substrings' argument > 0 include the remaining string as last element in the result", 0) \ - \ - M(Bool, allow_execute_multiif_columnar, true, "Allow execute multiIf function columnar", 0) \ - M(Bool, formatdatetime_f_prints_single_zero, false, "Formatter '%f' in function 'formatDateTime()' prints a single zero instead of six zeros if the formatted value has no fractional seconds.", 0) \ - M(Bool, formatdatetime_parsedatetime_m_is_month_name, true, "Formatter '%M' in functions 'formatDateTime()' and 'parseDateTime()' print/parse the month name instead of minutes.", 0) \ - M(Bool, parsedatetime_parse_without_leading_zeros, true, "Formatters '%c', '%l' and '%k' in function 'parseDateTime()' parse months and hours without leading zeros.", 0) \ - M(Bool, formatdatetime_format_without_leading_zeros, false, "Formatters '%c', '%l' and '%k' in function 'formatDateTime()' print months and hours without leading zeros.", 0) \ - \ - M(UInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.", 0) \ - M(Bool, throw_on_max_partitions_per_insert_block, true, "Used with max_partitions_per_insert_block. If true (default), an exception will be thrown when max_partitions_per_insert_block is reached. If false, details of the insert query reaching this limit with the number of partitions will be logged. This can be useful if you're trying to understand the impact on users when changing max_partitions_per_insert_block.", 0) \ - M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ - M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ - M(Bool, allow_drop_detached, false, "Allow ALTER TABLE ... DROP DETACHED PART[ITION] ... queries", 0) \ - M(UInt64, max_table_size_to_drop, 50000000000lu, "If size of a table is greater than this value (in bytes) than table could not be dropped with any DROP query.", 0) \ - M(UInt64, max_partition_size_to_drop, 50000000000lu, "Same as max_table_size_to_drop, but for the partitions.", 0) \ - \ - M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \ - M(UInt64, postgresql_connection_attempt_timeout, 2, "Connection timeout to PostgreSQL table engine and database engine in seconds.", 0) \ - M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \ - M(UInt64, postgresql_connection_pool_retries, 2, "Connection pool push/pop retries number for PostgreSQL table engine and database engine.", 0) \ - M(Bool, postgresql_connection_pool_auto_close_connection, false, "Close connection before returning connection to the pool.", 0) \ - M(UInt64, glob_expansion_max_elements, 1000, "Maximum number of allowed addresses (For external storages, table functions, etc).", 0) \ - M(UInt64, odbc_bridge_connection_pool_size, 16, "Connection pool size for each connection settings string in ODBC bridge.", 0) \ - M(Bool, odbc_bridge_use_connection_pooling, true, "Use connection pooling in ODBC bridge. If set to false, a new connection is created every time", 0) \ - \ - M(Seconds, distributed_replica_error_half_life, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD, "Time period reduces replica error counter by 2 times.", 0) \ - M(UInt64, distributed_replica_error_cap, DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT, "Max number of errors per replica, prevents piling up an incredible amount of errors if replica was offline for some time and allows it to be reconsidered in a shorter amount of time.", 0) \ - M(UInt64, distributed_replica_max_ignored_errors, 0, "Number of errors that will be ignored while choosing replicas", 0) \ - \ - M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ - \ - M(DefaultTableEngine, default_temporary_table_engine, DefaultTableEngine::Memory, "Default table engine used when ENGINE is not set in CREATE TEMPORARY statement.",0) \ - M(DefaultTableEngine, default_table_engine, DefaultTableEngine::MergeTree, "Default table engine used when ENGINE is not set in CREATE statement.",0) \ - M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ - M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \ - M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ - M(Bool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \ - M(Bool, optimize_trivial_approximate_count_query, false, "Use an approximate value for trivial count optimization of storages that support such estimations.", 0) \ - M(Bool, optimize_count_from_files, true, "Optimize counting rows from files in supported input formats", 0) \ - M(Bool, use_cache_for_count_from_files, true, "Use cache to count the number of rows in files", 0) \ - M(Bool, optimize_respect_aliases, true, "If it is set to true, it will respect aliases in WHERE/GROUP BY/ORDER BY, that will help with partition pruning/secondary indexes/optimize_aggregation_in_order/optimize_read_in_order/optimize_trivial_count", 0) \ - M(UInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ - M(Bool, enable_lightweight_delete, true, "Enable lightweight DELETE mutations for mergetree tables.", 0) ALIAS(allow_experimental_lightweight_delete) \ - M(UInt64, lightweight_deletes_sync, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes", 0) \ - M(Bool, apply_deleted_mask, true, "Enables filtering out rows deleted with lightweight DELETE. If disabled, a query will be able to read those rows. This is useful for debugging and \"undelete\" scenarios", 0) \ - M(Bool, optimize_normalize_count_variants, true, "Rewrite aggregate functions that semantically equals to count() as count().", 0) \ - M(Bool, optimize_injective_functions_inside_uniq, true, "Delete injective functions of one argument inside uniq*() functions.", 0) \ - M(Bool, rewrite_count_distinct_if_with_count_distinct_implementation, false, "Rewrite countDistinctIf with count_distinct_implementation configuration", 0) \ - M(Bool, convert_query_to_cnf, false, "Convert SELECT query to CNF", 0) \ - M(Bool, optimize_or_like_chain, false, "Optimize multiple OR LIKE into multiMatchAny. This optimization should not be enabled by default, because it defies index analysis in some cases.", 0) \ - M(Bool, optimize_arithmetic_operations_in_aggregate_functions, true, "Move arithmetic operations out of aggregation functions", 0) \ - M(Bool, optimize_redundant_functions_in_order_by, true, "Remove functions from ORDER BY if its argument is also in ORDER BY", 0) \ - M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \ - M(Bool, optimize_multiif_to_if, true, "Replace 'multiIf' with only one condition to 'if'.", 0) \ - M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \ - M(Bool, optimize_functions_to_subcolumns, true, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \ - M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0) \ - M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0) \ - M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \ - M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \ - M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \ - M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \ - M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views. Use true to always deduplicate in dependent tables.", 0) \ - M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \ - M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \ - M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \ - M(Bool, allow_materialized_view_with_bad_select, true, "Allow CREATE MATERIALIZED VIEW with SELECT query that references nonexistent tables or columns. It must still be syntactically valid. Doesn't apply to refreshable MVs. Doesn't apply if the MV schema needs to be inferred from the SELECT query (i.e. if the CREATE has no column list and no TO table). Can be used for creating MV before its source table.", 0) \ - M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \ - M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \ - M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \ - M(UInt64, max_parser_backtracks, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, "Maximum parser backtracking (how many times it tries different alternatives in the recursive descend parsing process).", 0) \ - M(UInt64, max_recursive_cte_evaluation_depth, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth", 0) \ - M(Bool, allow_settings_after_format_in_insert, false, "Allow SETTINGS after FORMAT, but note, that this is not always safe (note: this is a compatibility setting).", 0) \ - M(Seconds, periodic_live_view_refresh, 60, "Interval after which periodically refreshed live view is forced to refresh.", 0) \ - M(Bool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \ - M(Bool, allow_nondeterministic_mutations, false, "Allow non-deterministic functions in ALTER UPDATE/ALTER DELETE statements", 0) \ - M(Seconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \ - M(Bool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ - M(String, function_implementation, "", "Choose function implementation for specific target or variant (experimental). If empty enable all of them.", 0) \ - M(Bool, data_type_default_nullable, false, "Data types without NULL or NOT NULL will make Nullable", 0) \ - M(Bool, cast_keep_nullable, false, "CAST operator keep Nullable for result data type", 0) \ - M(Bool, cast_ipv4_ipv6_default_on_conversion_error, false, "CAST operator into IPv4, CAST operator into IPV6 type, toIPv4, toIPv6 functions will return default value instead of throwing exception on conversion error.", 0) \ - M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ - M(Bool, system_events_show_zero_values, false, "When querying system.events or system.metrics tables, include all metrics, even with zero values.", 0) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, MySQLDataTypesSupportList{}, "Defines how MySQL types are converted to corresponding ClickHouse types. A comma separated list in any combination of 'decimal', 'datetime64', 'date2Date32' or 'date2String'. decimal: convert NUMERIC and DECIMAL types to Decimal when precision allows it. datetime64: convert DATETIME and TIMESTAMP types to DateTime64 instead of DateTime when precision is not 0. date2Date32: convert DATE to Date32 instead of Date. Takes precedence over date2String. date2String: convert DATE to String instead of Date. Overridden by datetime64.", 0) \ - M(Bool, optimize_trivial_insert_select, false, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ - M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ - M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \ - M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \ - M(Bool, optimize_syntax_fuse_functions, false, "Allow apply fuse aggregating function. Available only with `allow_experimental_analyzer`", 0) \ - M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \ - M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \ - M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \ - M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \ - M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \ - M(Bool, optimize_use_projections, true, "Automatically choose projections to perform SELECT query", 0) ALIAS(allow_experimental_projection_optimization) \ - M(Bool, optimize_use_implicit_projections, true, "Automatically choose implicit projections to perform SELECT query", 0) \ - M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \ - M(String, force_optimize_projection_name, "", "If it is set to a non-empty string, check that this projection is used in the query at least once.", 0) \ - M(String, preferred_optimize_projection_name, "", "If it is set to a non-empty string, ClickHouse tries to apply specified projection", 0) \ - M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ - M(Bool, async_query_sending_for_remote, true, "Asynchronously create connections and send query to shards in remote query", 0) \ - M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \ - M(Bool, describe_extend_object_types, false, "Deduce concrete type of columns of type Object in DESCRIBE query", 0) \ - M(Bool, describe_include_subcolumns, false, "If true, subcolumns of all table columns will be included into result of DESCRIBE query", 0) \ - M(Bool, describe_include_virtual_columns, false, "If true, virtual columns of table will be included into result of DESCRIBE query", 0) \ - M(Bool, describe_compact_output, false, "If true, include only column names and types into result of DESCRIBE query", 0) \ - M(Bool, apply_mutations_on_fly, false, "Only available in ClickHouse Cloud", 0) \ - M(Bool, mutations_execute_nondeterministic_on_initiator, false, "If true nondeterministic function are executed on initiator and replaced to literals in UPDATE and DELETE queries", 0) \ - M(Bool, mutations_execute_subqueries_on_initiator, false, "If true scalar subqueries are executed on initiator and replaced to literals in UPDATE and DELETE queries", 0) \ - M(UInt64, mutations_max_literal_size_to_replace, 16384, "The maximum size of serialized literal in bytes to replace in UPDATE and DELETE queries", 0) \ - \ - M(Float, create_replicated_merge_tree_fault_injection_probability, 0.0f, "The probability of a fault injection during table creation after creating metadata in ZooKeeper", 0) \ - \ - M(Bool, use_query_cache, false, "Enable the query cache", 0) \ - M(Bool, enable_writes_to_query_cache, true, "Enable storing results of SELECT queries in the query cache", 0) \ - M(Bool, enable_reads_from_query_cache, true, "Enable reading results of SELECT queries from the query cache", 0) \ - M(QueryCacheNondeterministicFunctionHandling, query_cache_nondeterministic_function_handling, QueryCacheNondeterministicFunctionHandling::Throw, "How the query cache handles queries with non-deterministic functions, e.g. now()", 0) \ - M(QueryCacheSystemTableHandling, query_cache_system_table_handling, QueryCacheSystemTableHandling::Throw, "How the query cache handles queries against system tables, i.e. tables in databases 'system.*' and 'information_schema.*'", 0) \ - M(UInt64, query_cache_max_size_in_bytes, 0, "The maximum amount of memory (in bytes) the current user may allocate in the query cache. 0 means unlimited. ", 0) \ - M(UInt64, query_cache_max_entries, 0, "The maximum number of query results the current user may store in the query cache. 0 means unlimited.", 0) \ - M(UInt64, query_cache_min_query_runs, 0, "Minimum number a SELECT query must run before its result is stored in the query cache", 0) \ - M(Milliseconds, query_cache_min_query_duration, 0, "Minimum time in milliseconds for a query to run for its result to be stored in the query cache.", 0) \ - M(Bool, query_cache_compress_entries, true, "Compress cache entries.", 0) \ - M(Bool, query_cache_squash_partial_results, true, "Squash partial result blocks to blocks of size 'max_block_size'. Reduces performance of inserts into the query cache but improves the compressability of cache entries.", 0) \ - M(Seconds, query_cache_ttl, 60, "After this time in seconds entries in the query cache become stale", 0) \ - M(Bool, query_cache_share_between_users, false, "Allow other users to read entry in the query cache", 0) \ - M(String, query_cache_tag, "", "A string which acts as a label for query cache entries. The same queries with different tags are considered different by the query cache.", 0) \ - M(Bool, enable_sharing_sets_for_mutations, true, "Allow sharing set objects build for IN subqueries between different tasks of the same mutation. This reduces memory usage and CPU consumption", 0) \ - \ - M(Bool, optimize_rewrite_sum_if_to_count_if, true, "Rewrite sumIf() and sum(if()) function countIf() function when logically equivalent", 0) \ - M(Bool, optimize_rewrite_aggregate_function_with_if, true, "Rewrite aggregate functions with if expression as argument when logically equivalent. For example, avg(if(cond, col, null)) can be rewritten to avgIf(cond, col)", 0) \ - M(Bool, optimize_rewrite_array_exists_to_has, false, "Rewrite arrayExists() functions to has() when logically equivalent. For example, arrayExists(x -> x = 1, arr) can be rewritten to has(arr, 1)", 0) \ - M(UInt64, insert_shard_id, 0, "If non zero, when insert into a distributed table, the data will be inserted into the shard `insert_shard_id` synchronously. Possible values range from 1 to `shards_number` of corresponding distributed table", 0) \ - \ - M(Bool, collect_hash_table_stats_during_aggregation, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ - M(UInt64, max_size_to_preallocate_for_aggregation, 100'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before aggregation", 0) \ - \ - M(Bool, collect_hash_table_stats_during_joins, true, "Enable collecting hash table statistics to optimize memory allocation", 0) \ - M(UInt64, max_size_to_preallocate_for_joins, 100'000'000, "For how many elements it is allowed to preallocate space in all hash tables in total before join", 0) \ - \ - M(Bool, kafka_disable_num_consumers_limit, false, "Disable limit on kafka_num_consumers that depends on the number of available CPU cores", 0) \ - M(Bool, allow_experimental_kafka_offsets_storage_in_keeper, false, "Allow experimental feature to store Kafka related offsets in ClickHouse Keeper. When enabled a ClickHouse Keeper path and replica name can be specified to the Kafka table engine. As a result instead of the regular Kafka engine, a new type of storage engine will be used that stores the committed offsets primarily in ClickHouse Keeper", 0) \ - M(Bool, enable_software_prefetch_in_aggregation, true, "Enable use of software prefetch in aggregation", 0) \ - M(Bool, allow_aggregate_partitions_independently, false, "Enable independent aggregation of partitions on separate threads when partition key suits group by key. Beneficial when number of partitions close to number of cores and partitions have roughly the same size", 0) \ - M(Bool, force_aggregate_partitions_independently, false, "Force the use of optimization when it is applicable, but heuristics decided not to use it", 0) \ - M(UInt64, max_number_of_partitions_for_independent_aggregation, 128, "Maximal number of partitions in table to apply optimization", 0) \ - M(Float, min_hit_rate_to_use_consecutive_keys_optimization, 0.5, "Minimal hit rate of a cache which is used for consecutive keys optimization in aggregation to keep it enabled", 0) \ - \ - M(Bool, engine_file_empty_if_not_exists, false, "Allows to select data from a file engine table without file", 0) \ - M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ - M(Bool, engine_file_allow_create_multiple_files, false, "Enables or disables creating a new file on each insert in file engine tables if format has suffix.", 0) \ - M(Bool, engine_file_skip_empty_files, false, "Allows to skip empty files in file table engine", 0) \ - M(Bool, engine_url_skip_empty_files, false, "Allows to skip empty files in the URL table engine", 0) \ - M(Bool, enable_url_encoding, true, " Allows to enable/disable decoding/encoding path in URI in the URL table engine", 0) \ - M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ - M(Bool, database_replicated_enforce_synchronous_settings, false, "Enforces synchronous waiting for some queries (see also database_atomic_wait_for_drop_and_detach_synchronously, mutation_sync, alter_sync). Not recommended to enable these settings.", 0) \ - M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \ - M(Bool, database_replicated_always_detach_permanently, false, "Execute DETACH TABLE as DETACH TABLE PERMANENTLY if database engine is Replicated", 0) \ - M(Bool, database_replicated_allow_only_replicated_engine, false, "Allow to create only Replicated tables in database with engine Replicated", 0) \ - M(UInt64, database_replicated_allow_replicated_engine_arguments, 0, "0 - Don't allow to explicitly specify ZooKeeper path and replica name for *MergeTree tables in Replicated databases. 1 - Allow. 2 - Allow, but ignore the specified path and use default one instead.", 0) \ - M(UInt64, database_replicated_allow_explicit_uuid, 0, "0 - Don't allow to explicitly specify UUIDs for tables in Replicated databases. 1 - Allow. 2 - Allow, but ignore the specified UUID and generate a random one instead.", 0) \ - M(Bool, database_replicated_allow_heavy_create, false, "Allow long-running DDL queries (CREATE AS SELECT and POPULATE) in Replicated database engine. Note that it can block DDL queue for a long time.", 0) \ - M(Bool, cloud_mode, false, "Only available in ClickHouse Cloud", 0) \ - M(UInt64, cloud_mode_engine, 1, "Only available in ClickHouse Cloud", 0) \ - M(DistributedDDLOutputMode, distributed_ddl_output_mode, DistributedDDLOutputMode::THROW, "Format of distributed DDL query result, one of: 'none', 'throw', 'null_status_on_timeout', 'never_throw', 'none_only_active', 'throw_only_active', 'null_status_on_timeout_only_active'", 0) \ - M(UInt64, distributed_ddl_entry_format_version, 5, "Compatibility version of distributed DDL (ON CLUSTER) queries", 0) \ - \ - M(UInt64, external_storage_max_read_rows, 0, "Limit maximum number of rows when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializedMySQL. If equal to 0, this setting is disabled", 0) \ - M(UInt64, external_storage_max_read_bytes, 0, "Limit maximum number of bytes when table with external engine should flush history data. Now supported only for MySQL table engine, database engine, dictionary and MaterializedMySQL. If equal to 0, this setting is disabled", 0) \ - M(UInt64, external_storage_connect_timeout_sec, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connect timeout in seconds. Now supported only for MySQL", 0) \ - M(UInt64, external_storage_rw_timeout_sec, DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, "Read/write timeout in seconds. Now supported only for MySQL", 0) \ - \ - M(SetOperationMode, union_default_mode, SetOperationMode::Unspecified, "Set default mode in UNION query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without mode will throw exception.", 0) \ - M(SetOperationMode, intersect_default_mode, SetOperationMode::ALL, "Set default mode in INTERSECT query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without mode will throw exception.", 0) \ - M(SetOperationMode, except_default_mode, SetOperationMode::ALL, "Set default mode in EXCEPT query. Possible values: empty string, 'ALL', 'DISTINCT'. If empty, query without mode will throw exception.", 0) \ - M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ - M(Bool, optimize_injective_functions_in_group_by, true, "Replaces injective functions by it's arguments in GROUP BY section", 0) \ - M(Bool, optimize_group_by_function_keys, true, "Eliminates functions of other keys in GROUP BY section", 0) \ - M(Bool, optimize_group_by_constant_keys, true, "Optimize GROUP BY when all keys in block are constant", 0) \ - M(Bool, legacy_column_name_of_tuple_literal, false, "List all names of element of large tuple literals in their column names instead of hash. This settings exists only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher.", 0) \ - M(Bool, enable_named_columns_in_function_tuple, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers.", 0) \ - \ - M(Bool, query_plan_enable_optimizations, true, "Globally enable/disable query optimization at the query plan level", 0) \ - M(UInt64, query_plan_max_optimizations_to_apply, 10000, "Limit the total number of optimizations applied to query plan. If zero, ignored. If limit reached, throw exception", 0) \ - M(Bool, query_plan_lift_up_array_join, true, "Allow to move array joins up in the query plan", 0) \ - M(Bool, query_plan_push_down_limit, true, "Allow to move LIMITs down in the query plan", 0) \ - M(Bool, query_plan_split_filter, true, "Allow to split filters in the query plan", 0) \ - M(Bool, query_plan_merge_expressions, true, "Allow to merge expressions in the query plan", 0) \ - M(Bool, query_plan_merge_filters, false, "Allow to merge filters in the query plan", 0) \ - M(Bool, query_plan_filter_push_down, true, "Allow to push down filter by predicate query plan step", 0) \ - M(Bool, query_plan_convert_outer_join_to_inner_join, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values", 0) \ - M(Bool, query_plan_optimize_prewhere, true, "Allow to push down filter to PREWHERE expression for supported storages", 0) \ - M(Bool, query_plan_execute_functions_after_sorting, true, "Allow to re-order functions after sorting", 0) \ - M(Bool, query_plan_reuse_storage_ordering_for_window_functions, true, "Allow to use the storage sorting for window functions", 0) \ - M(Bool, query_plan_lift_up_union, true, "Allow to move UNIONs up so that more parts of the query plan can be optimized", 0) \ - M(Bool, query_plan_read_in_order, true, "Use query plan for read-in-order optimization", 0) \ - M(Bool, query_plan_aggregation_in_order, true, "Use query plan for aggregation-in-order optimization", 0) \ - M(Bool, query_plan_remove_redundant_sorting, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries", 0) \ - M(Bool, query_plan_remove_redundant_distinct, true, "Remove redundant Distinct step in query plan", 0) \ - M(Bool, query_plan_enable_multithreading_after_window_functions, true, "Enable multithreading after evaluating window functions to allow parallel stream processing", 0) \ - M(UInt64, regexp_max_matches_per_row, 1000, "Max matches of any single regexp per row, used to safeguard 'extractAllGroupsHorizontal' against consuming too much memory with greedy RE.", 0) \ - \ - M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ - M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ - \ - M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function `range` per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \ - M(UInt64, function_sleep_max_microseconds_per_block, 3000000, "Maximum number of microseconds the function `sleep` is allowed to sleep for each block. If a user called it with a larger value, it throws an exception. It is a safety threshold.", 0) \ - M(UInt64, function_visible_width_behavior, 1, "The version of `visibleWidth` behavior. 0 - only count the number of code points; 1 - correctly count zero-width and combining characters, count full-width characters as two, estimate the tab width, count delete characters.", 0) \ - M(ShortCircuitFunctionEvaluation, short_circuit_function_evaluation, ShortCircuitFunctionEvaluation::ENABLE, "Setting for short-circuit function evaluation configuration. Possible values: 'enable' - use short-circuit function evaluation for functions that are suitable for it, 'disable' - disable short-circuit function evaluation, 'force_enable' - use short-circuit function evaluation for all functions.", 0) \ - \ - M(LocalFSReadMethod, storage_file_read_method, LocalFSReadMethod::pread, "Method of reading data from storage file, one of: read, pread, mmap. The mmap method does not apply to clickhouse-server (it's intended for clickhouse-local).", 0) \ - M(String, local_filesystem_read_method, "pread_threadpool", "Method of reading data from local filesystem, one of: read, pread, mmap, io_uring, pread_threadpool. The 'io_uring' method is experimental and does not work for Log, TinyLog, StripeLog, File, Set and Join, and other tables with append-able files in presence of concurrent reads and writes.", 0) \ - M(String, remote_filesystem_read_method, "threadpool", "Method of reading data from remote filesystem, one of: read, threadpool.", 0) \ - M(Bool, local_filesystem_read_prefetch, false, "Should use prefetching when reading data from local filesystem.", 0) \ - M(Bool, remote_filesystem_read_prefetch, true, "Should use prefetching when reading data from remote filesystem.", 0) \ - M(Int64, read_priority, 0, "Priority to read data from local filesystem or remote filesystem. Only supported for 'pread_threadpool' method for local filesystem and for `threadpool` method for remote filesystem.", 0) \ - M(UInt64, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \ - M(UInt64, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \ - M(UInt64, remote_read_min_bytes_for_seek, 4 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead of read with ignore.", 0) \ - M(UInt64, merge_tree_min_bytes_per_task_for_remote_reading, 2 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes to read per task.", 0) ALIAS(filesystem_prefetch_min_bytes_for_single_read_task) \ - M(Bool, merge_tree_use_const_size_tasks_for_remote_reading, true, "Whether to use constant size tasks for reading from a remote table.", 0) \ - M(Bool, merge_tree_determine_task_size_by_prewhere_columns, true, "Whether to use only prewhere columns size to determine reading task size.", 0) \ - M(UInt64, merge_tree_compact_parts_min_granules_to_multibuffer_read, 16, "Only available in ClickHouse Cloud", 0) \ - \ - M(Bool, async_insert, false, "If true, data from INSERT query is stored in queue and later flushed to table in background. If wait_for_async_insert is false, INSERT query is processed almost instantly, otherwise client will wait until data will be flushed to table", 0) \ - M(Bool, wait_for_async_insert, true, "If true wait for processing of asynchronous insertion", 0) \ - M(Seconds, wait_for_async_insert_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "Timeout for waiting for processing asynchronous insertion", 0) \ - M(UInt64, async_insert_max_data_size, 10485760, "Maximum size in bytes of unparsed data collected per query before being inserted", 0) \ - M(UInt64, async_insert_max_query_number, 450, "Maximum number of insert queries before being inserted", 0) \ - M(Milliseconds, async_insert_poll_timeout_ms, 10, "Timeout for polling data from asynchronous insert queue", 0) \ - M(Bool, async_insert_use_adaptive_busy_timeout, true, "If it is set to true, use adaptive busy timeout for asynchronous inserts", 0) \ - M(Milliseconds, async_insert_busy_timeout_min_ms, 50, "If auto-adjusting is enabled through async_insert_use_adaptive_busy_timeout, minimum time to wait before dumping collected data per query since the first data appeared. It also serves as the initial value for the adaptive algorithm", 0) \ - M(Milliseconds, async_insert_busy_timeout_max_ms, 200, "Maximum time to wait before dumping collected data per query since the first data appeared.", 0) ALIAS(async_insert_busy_timeout_ms) \ - M(Double, async_insert_busy_timeout_increase_rate, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases", 0) \ - M(Double, async_insert_busy_timeout_decrease_rate, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases", 0) \ - \ - M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ - M(UInt64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ - M(Bool, enable_filesystem_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must be done via disk config), but allows to bypass cache for some queries if intended", 0) \ - M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \ - M(Bool, enable_filesystem_cache_log, false, "Allows to record the filesystem caching log for each query", 0) \ - M(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, "Allow to use the filesystem cache in passive mode - benefit from the existing cache entries, but don't put more entries into the cache. If you set this setting for heavy ad-hoc queries and leave it disabled for short real-time queries, this will allows to avoid cache threshing by too heavy queries and to improve the overall system efficiency.", 0) \ - M(Bool, skip_download_if_exceeds_query_cache, true, "Skip download from remote filesystem if exceeds query cache size", 0) \ - M(UInt64, filesystem_cache_max_download_size, (128UL * 1024 * 1024 * 1024), "Max remote filesystem cache size that can be downloaded by a single query", 0) \ - M(Bool, throw_on_error_from_cache_on_write_operations, false, "Ignore error from cache when caching on write operations (INSERT, merges)", 0) \ - M(UInt64, filesystem_cache_segments_batch_size, 20, "Limit on size of a single batch of file segments that a read buffer can request from cache. Too low value will lead to excessive requests to cache, too large may slow down eviction from cache", 0) \ - M(UInt64, filesystem_cache_reserve_space_wait_lock_timeout_milliseconds, 1000, "Wait time to lock cache for space reservation in filesystem cache", 0) \ - M(UInt64, temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds, (10 * 60 * 1000), "Wait time to lock cache for space reservation for temporary data in filesystem cache", 0) \ - \ - M(Bool, use_page_cache_for_disks_without_file_cache, false, "Use userspace page cache for remote disks that don't have filesystem cache enabled.", 0) \ - M(Bool, read_from_page_cache_if_exists_otherwise_bypass_cache, false, "Use userspace page cache in passive mode, similar to read_from_filesystem_cache_if_exists_otherwise_bypass_cache.", 0) \ - M(Bool, page_cache_inject_eviction, false, "Userspace page cache will sometimes invalidate some pages at random. Intended for testing.", 0) \ - \ - M(Bool, load_marks_asynchronously, false, "Load MergeTree marks asynchronously", 0) \ - M(Bool, enable_filesystem_read_prefetches_log, false, "Log to system.filesystem prefetch_log during query. Should be used only for testing or debugging, not recommended to be turned on by default", 0) \ - M(Bool, allow_prefetched_read_pool_for_remote_filesystem, true, "Prefer prefetched threadpool if all parts are on remote filesystem", 0) \ - M(Bool, allow_prefetched_read_pool_for_local_filesystem, false, "Prefer prefetched threadpool if all parts are on local filesystem", 0) \ - \ - M(UInt64, prefetch_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the prefetch buffer to read from the filesystem.", 0) \ - M(UInt64, filesystem_prefetch_step_bytes, 0, "Prefetch step in bytes. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \ - M(UInt64, filesystem_prefetch_step_marks, 0, "Prefetch step in marks. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \ - M(UInt64, filesystem_prefetch_max_memory_usage, "1Gi", "Maximum memory usage for prefetches.", 0) \ - M(UInt64, filesystem_prefetches_limit, 200, "Maximum number of prefetches. Zero means unlimited. A setting `filesystem_prefetches_max_memory_usage` is more recommended if you want to limit the number of prefetches", 0) \ - \ - M(UInt64, use_structure_from_insertion_table_in_table_functions, 2, "Use structure from insertion table instead of schema inference from data. Possible values: 0 - disabled, 1 - enabled, 2 - auto", 0) \ - \ - M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ - M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \ - M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \ - \ - M(Bool, force_remove_data_recursively_on_drop, false, "Recursively remove data on DROP query. Avoids 'Directory not empty' error, but may silently remove detached data", 0) \ - M(Bool, check_table_dependencies, true, "Check that DDL query (such as DROP TABLE or RENAME) will not break dependencies", 0) \ - M(Bool, check_referential_table_dependencies, false, "Check that DDL query (such as DROP TABLE or RENAME) will not break referential dependencies", 0) \ - M(Bool, use_local_cache_for_remote_storage, true, "Use local cache for remote storage like HDFS or S3, it's used for remote table engine only", 0) \ - \ - M(Bool, allow_unrestricted_reads_from_keeper, false, "Allow unrestricted (without condition on path) reads from system.zookeeper table, can be handy, but is not safe for zookeeper", 0) \ - M(Bool, allow_deprecated_database_ordinary, false, "Allow to create databases with deprecated Ordinary engine", 0) \ - M(Bool, allow_deprecated_syntax_for_merge_tree, false, "Allow to create *MergeTree tables with deprecated engine definition syntax", 0) \ - M(Bool, allow_asynchronous_read_from_io_pool_for_merge_tree, false, "Use background I/O pool to read from MergeTree tables. This setting may increase performance for I/O bound queries", 0) \ - M(UInt64, max_streams_for_merge_tree_reading, 0, "If is not zero, limit the number of reading streams for MergeTree table.", 0) \ - \ - M(Bool, force_grouping_standard_compatibility, true, "Make GROUPING function to return 1 when argument is not used as an aggregation key", 0) \ - \ - M(Bool, schema_inference_use_cache_for_file, true, "Use cache in schema inference while using file table function", 0) \ - M(Bool, schema_inference_use_cache_for_s3, true, "Use cache in schema inference while using s3 table function", 0) \ - M(Bool, schema_inference_use_cache_for_azure, true, "Use cache in schema inference while using azure table function", 0) \ - M(Bool, schema_inference_use_cache_for_hdfs, true, "Use cache in schema inference while using hdfs table function", 0) \ - M(Bool, schema_inference_use_cache_for_url, true, "Use cache in schema inference while using url table function", 0) \ - M(Bool, schema_inference_cache_require_modification_time_for_url, true, "Use schema from cache for URL with last modification time validation (for URLs with Last-Modified header)", 0) \ - \ - M(String, compatibility, "", "Changes other settings according to provided ClickHouse version. If we know that we changed some behaviour in ClickHouse by changing some settings in some version, this compatibility setting will control these settings", 0) \ - \ - M(Map, additional_table_filters, "", "Additional filter expression which would be applied after reading from specified table. Syntax: {'table1': 'expression', 'database.table2': 'expression'}", 0) \ - M(String, additional_result_filter, "", "Additional filter expression which would be applied to query result", 0) \ - \ - M(String, workload, "default", "Name of workload to be used to access resources", 0) \ - M(Milliseconds, storage_system_stack_trace_pipe_read_timeout_ms, 100, "Maximum time to read from a pipe for receiving information from the threads when querying the `system.stack_trace` table. This setting is used for testing purposes and not meant to be changed by users.", 0) \ - \ - M(String, rename_files_after_processing, "", "Rename successfully processed files according to the specified pattern; Pattern can include the following placeholders: `%a` (full original file name), `%f` (original filename without extension), `%e` (file extension with dot), `%t` (current timestamp in µs), and `%%` (% sign)", 0) \ - \ - M(Bool, parallelize_output_from_storages, true, "Parallelize output for reading step from storage. It allows parallelization of query processing right after reading from storage if possible", 0) \ - M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ - M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ - M(Bool, throw_if_no_data_to_insert, true, "Allows or forbids empty INSERTs, enabled by default (throws an error on an empty insert)", 0) \ - M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ - M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ - M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \ - M(UInt64, keeper_max_retries, 10, "Max retries for general keeper operations", 0) \ - M(UInt64, keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for general keeper operations", 0) \ - M(UInt64, keeper_retry_max_backoff_ms, 5000, "Max backoff timeout for general keeper operations", 0) \ - M(UInt64, insert_keeper_max_retries, 20, "Max retries for keeper operations during insert", 0) \ - M(UInt64, insert_keeper_retry_initial_backoff_ms, 100, "Initial backoff timeout for keeper operations during insert", 0) \ - M(UInt64, insert_keeper_retry_max_backoff_ms, 10000, "Max backoff timeout for keeper operations during insert", 0) \ - M(Float, insert_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during insert. Valid value is in interval [0.0f, 1.0f]", 0) \ - M(UInt64, insert_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ - M(Bool, force_aggregation_in_order, false, "The setting is used by the server itself to support distributed queries. Do not change it manually, because it will break normal operations. (Forces use of aggregation in order on remote nodes during distributed aggregation).", IMPORTANT) \ - M(UInt64, http_max_request_param_data_size, 10_MiB, "Limit on size of request data used as a query parameter in predefined HTTP requests.", 0) \ - M(Bool, function_json_value_return_type_allow_nullable, false, "Allow function JSON_VALUE to return nullable type.", 0) \ - M(Bool, function_json_value_return_type_allow_complex, false, "Allow function JSON_VALUE to return complex type, such as: struct, array, map.", 0) \ - M(Bool, use_with_fill_by_sorting_prefix, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently", 0) \ - M(Bool, optimize_uniq_to_count, true, "Rewrite uniq and its variants(except uniqUpTo) to count if subquery has distinct or group by clause.", 0) \ - M(Bool, use_variant_as_common_type, false, "Use Variant as a result type for if/multiIf in case when there is no common type for arguments", 0) \ - M(Bool, enable_order_by_all, true, "Enable sorting expression ORDER BY ALL.", 0) \ - M(Float, ignore_drop_queries_probability, 0, "If enabled, server will ignore all DROP table queries with specified probability (for Memory and JOIN engines it will replcase DROP to TRUNCATE). Used for testing purposes", 0) \ - M(Bool, traverse_shadow_remote_data_paths, false, "Traverse shadow directory when query system.remote_data_paths", 0) \ - M(Bool, geo_distance_returns_float64_on_float64_arguments, true, "If all four arguments to `geoDistance`, `greatCircleDistance`, `greatCircleAngle` functions are Float64, return Float64 and use double precision for internal calculations. In previous ClickHouse versions, the functions always returned Float32.", 0) \ - M(Bool, allow_get_client_http_header, false, "Allow to use the function `getClientHTTPHeader` which lets to obtain a value of an the current HTTP request's header. It is not enabled by default for security reasons, because some headers, such as `Cookie`, could contain sensitive info. Note that the `X-ClickHouse-*` and `Authentication` headers are always restricted and cannot be obtained with this function.", 0) \ - M(Bool, cast_string_to_dynamic_use_inference, false, "Use types inference during String to Dynamic conversion", 0) \ - M(Bool, enable_blob_storage_log, true, "Write information about blob storage operations to system.blob_storage_log table", 0) \ - M(Bool, use_json_alias_for_old_object_type, false, "When enabled, JSON type alias will create old experimental Object type instead of a new JSON type", 0) \ - M(Bool, allow_create_index_without_type, false, "Allow CREATE INDEX query without TYPE. Query will be ignored. Made for SQL compatibility tests.", 0) \ - M(Bool, create_index_ignore_unique, false, "Ignore UNIQUE keyword in CREATE UNIQUE INDEX. Made for SQL compatibility tests.", 0) \ - M(Bool, print_pretty_type_names, true, "Print pretty type names in the DESCRIBE query and `toTypeName` function, as well as in the `SHOW CREATE TABLE` query and the `formatQuery` function.", 0) \ - M(Bool, create_table_empty_primary_key_by_default, false, "Allow to create *MergeTree tables with empty primary key when ORDER BY and PRIMARY KEY not specified", 0) \ - M(Bool, allow_named_collection_override_by_default, true, "Allow named collections' fields override by default.", 0) \ - M(SQLSecurityType, default_normal_view_sql_security, SQLSecurityType::INVOKER, "Allows to set a default value for SQL SECURITY option when creating a normal view.", 0) \ - M(SQLSecurityType, default_materialized_view_sql_security, SQLSecurityType::DEFINER, "Allows to set a default value for SQL SECURITY option when creating a materialized view.", 0) \ - M(String, default_view_definer, "CURRENT_USER", "Allows to set a default value for DEFINER option when creating view.", 0) \ - M(UInt64, cache_warmer_threads, 4, "Only available in ClickHouse Cloud. Number of background threads for speculatively downloading new data parts into file cache, when cache_populated_by_fetch is enabled. Zero to disable.", 0) \ - M(Int64, ignore_cold_parts_seconds, 0, "Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.", 0) \ - M(Int64, prefer_warmed_unmerged_parts_seconds, 0, "Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm.", 0) \ - M(Bool, iceberg_engine_ignore_schema_evolution, false, "Ignore schema evolution in Iceberg table engine and read all data using latest schema saved on table creation. Note that it can lead to incorrect result", 0) \ - M(Bool, allow_deprecated_error_prone_window_functions, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)", 0) \ - M(Bool, allow_deprecated_snowflake_conversion_functions, false, "Enables deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake.", 0) \ - M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \ - M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \ - M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \ - M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \ - M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \ - M(Bool, create_if_not_exists, false, "Enable IF NOT EXISTS for CREATE statements by default", 0) \ - \ - \ - /* ###################################### */ \ - /* ######## EXPERIMENTAL FEATURES ####### */ \ - /* ###################################### */ \ - M(Bool, allow_experimental_materialized_postgresql_table, false, "Allows to use the MaterializedPostgreSQL table engine. Disabled by default, because this feature is experimental", 0) \ - M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ - M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \ - M(Bool, allow_experimental_hash_functions, false, "Enable experimental hash functions", 0) \ - M(Bool, allow_experimental_object_type, false, "Allow Object and JSON data types", 0) \ - M(Bool, allow_experimental_time_series_table, false, "Allows experimental TimeSeries table engine", 0) \ - M(Bool, allow_experimental_vector_similarity_index, false, "Allow experimental vector similarity index", 0) \ - M(Bool, allow_experimental_variant_type, false, "Allow Variant data type", 0) \ - M(Bool, allow_experimental_dynamic_type, false, "Allow Dynamic data type", 0) \ - M(Bool, allow_experimental_json_type, false, "Allow JSON data type", 0) \ - M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ - M(UInt64, max_limit_for_ann_queries, 1'000'000, "SELECT queries with LIMIT bigger than this setting cannot use ANN indexes. Helps to prevent memory overflows in ANN search indexes.", 0) \ - M(Bool, throw_on_unsupported_query_inside_transaction, true, "Throw exception if unsupported query is used inside transaction", 0) \ - M(TransactionsWaitCSNMode, wait_changes_become_visible_after_commit_mode, TransactionsWaitCSNMode::WAIT_UNKNOWN, "Wait for committed changes to become actually visible in the latest snapshot", 0) \ - M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ - M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ - M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ - M(Int32, join_to_sort_minimum_perkey_rows, 40, "The lower limit of per-key average rows in the right table to determine whether to rerange the right table by key in left or inner join. This setting ensures that the optimization is not applied for sparse table keys", 0) \ - M(Int32, join_to_sort_maximum_table_rows, 10000, "The maximum number of rows in the right table to determine whether to rerange the right table by key in left or inner join.", 0) \ - M(Bool, allow_experimental_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join.", 0) \ - M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ - M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ - \ - M(Bool, allow_statistics_optimize, false, "Allows using statistics to optimize queries", 0) ALIAS(allow_statistic_optimize) \ - M(Bool, allow_experimental_statistics, false, "Allows using statistics", 0) ALIAS(allow_experimental_statistic) \ - \ - /* Parallel replicas */ \ - M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ - M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ - M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ - M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \ - M(UInt64, parallel_replicas_custom_key_range_lower, 0, "Lower bound for the universe that the parallel replicas custom range filter is calculated over", 0) \ - M(UInt64, parallel_replicas_custom_key_range_upper, 0, "Upper bound for the universe that the parallel replicas custom range filter is calculated over. A value of 0 disables the upper bound, setting it to the max value of the custom key expression", 0) \ - M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \ - M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \ - M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \ - M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \ - M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \ - M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \ - M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \ - M(UInt64, parallel_replicas_mark_segment_size, 0, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384]", 0) \ - M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as ' :: ' if archive has correct extension", 0) \ - M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \ - \ - M(Bool, allow_experimental_inverted_index, false, "If it is set to true, allow to use experimental inverted index.", 0) \ - M(Bool, allow_experimental_full_text_index, false, "If it is set to true, allow to use experimental full-text index.", 0) \ - \ - M(Bool, allow_experimental_join_condition, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y.", 0) \ - \ - M(Bool, allow_experimental_analyzer, true, "Allow new query analyzer.", IMPORTANT) ALIAS(enable_analyzer) \ - M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \ - \ - M(Bool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \ - M(Seconds, live_view_heartbeat_interval, 15, "The heartbeat interval in seconds to indicate live query is alive.", 0) \ - M(UInt64, max_live_view_insert_blocks_before_refresh, 64, "Limit maximum number of inserted blocks after which mergeable blocks are dropped and query is re-executed.", 0) \ - \ - M(Bool, allow_experimental_window_view, false, "Enable WINDOW VIEW. Not mature enough.", 0) \ - M(Seconds, window_view_clean_interval, 60, "The clean interval of window view in seconds to free outdated data.", 0) \ - M(Seconds, window_view_heartbeat_interval, 15, "The heartbeat interval in seconds to indicate watch query is alive.", 0) \ - M(Seconds, wait_for_window_view_fire_signal_timeout, 10, "Timeout for waiting for window view fire signal in event time processing", 0) \ - \ - M(Bool, allow_experimental_refreshable_materialized_view, false, "Allow refreshable materialized views (CREATE MATERIALIZED VIEW REFRESH ...).", 0) \ - M(Bool, stop_refreshable_materialized_views_on_startup, false, "On server startup, prevent scheduling of refreshable materialized views, as if with SYSTEM STOP VIEWS. You can manually start them with SYSTEM START VIEWS or SYSTEM START VIEW afterwards. Also applies to newly created views. Has no effect on non-refreshable materialized views.", 0) \ - \ - M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \ - M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \ - /** Experimental feature for moving data between shards. */ \ - M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \ - - /** End of experimental features */ - -// End of COMMON_SETTINGS -// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS, move obsolete settings to OBSOLETE_SETTINGS and obsolete format settings to OBSOLETE_FORMAT_SETTINGS. - -#define MAKE_OBSOLETE(M, TYPE, NAME, DEFAULT) \ - M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE) - -/// NOTE: ServerSettings::loadSettingsFromConfig() should be updated to include this settings -#define MAKE_DEPRECATED_BY_SERVER_CONFIG(M, TYPE, NAME, DEFAULT) \ - M(TYPE, NAME, DEFAULT, "User-level setting is deprecated, and it must be defined in the server configuration instead.", BaseSettingsHelpers::Flags::OBSOLETE) - -#define OBSOLETE_SETTINGS(M, ALIAS) \ - /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ - MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \ - MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \ - MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \ - MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_database_atomic, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_bigint_types, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_window_functions, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_geo_types, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_query_cache, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_alter_materialized_view_structure, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_shared_merge_tree, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_database_replicated, true) \ - \ - MAKE_OBSOLETE(M, Milliseconds, async_insert_stale_timeout_ms, 0) \ - MAKE_OBSOLETE(M, StreamingHandleErrorMode, handle_kafka_error_mode, StreamingHandleErrorMode::DEFAULT) \ - MAKE_OBSOLETE(M, Bool, database_replicated_ddl_output, true) \ - MAKE_OBSOLETE(M, UInt64, replication_alter_columns_timeout, 60) \ - MAKE_OBSOLETE(M, UInt64, odbc_max_field_size, 0) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_map_type, true) \ - MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_temporary_directories_interval_seconds, 60) \ - MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_parts_interval_seconds, 1) \ - MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \ - MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \ - MAKE_OBSOLETE(M, Bool, use_mysql_types_in_show_columns, false) \ - MAKE_OBSOLETE(M, Bool, s3queue_allow_experimental_sharded_mode, false) \ - /* moved to config.xml: see also src/Core/ServerSettings.h */ \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_buffer_flush_schedule_pool_size, 16) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_pool_size, 16) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, Float, background_merges_mutations_concurrency_ratio, 2) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_move_pool_size, 8) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_fetches_pool_size, 8) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_common_pool_size, 8) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_schedule_pool_size, 128) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_message_broker_schedule_pool_size, 16) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, background_distributed_schedule_pool_size, 16) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_remote_read_network_bandwidth_for_server, 0) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_remote_write_network_bandwidth_for_server, 0) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, async_insert_threads, 16) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_fetches_network_bandwidth_for_server, 0) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_replicated_sends_network_bandwidth_for_server, 0) \ - MAKE_DEPRECATED_BY_SERVER_CONFIG(M, UInt64, max_entries_for_hash_table_stats, 10'000) \ - /* ---- */ \ - MAKE_OBSOLETE(M, DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic) \ - MAKE_OBSOLETE(M, UInt64, max_pipeline_depth, 0) \ - MAKE_OBSOLETE(M, Seconds, temporary_live_view_timeout, 1) \ - MAKE_OBSOLETE(M, Milliseconds, async_insert_cleanup_timeout_ms, 1000) \ - MAKE_OBSOLETE(M, Bool, optimize_fuse_sum_count_avg, 0) \ - MAKE_OBSOLETE(M, Seconds, drain_timeout, 3) \ - MAKE_OBSOLETE(M, UInt64, backup_threads, 16) \ - MAKE_OBSOLETE(M, UInt64, restore_threads, 16) \ - MAKE_OBSOLETE(M, Bool, optimize_duplicate_order_by_and_distinct, false) \ - MAKE_OBSOLETE(M, UInt64, parallel_replicas_min_number_of_granules_to_enable, 0) \ - MAKE_OBSOLETE(M, Bool, query_plan_optimize_projection, true) \ - MAKE_OBSOLETE(M, Bool, query_cache_store_results_of_queries_with_nondeterministic_functions, false) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_annoy_index, false) \ - MAKE_OBSOLETE(M, UInt64, max_threads_for_annoy_index_creation, 4) \ - MAKE_OBSOLETE(M, Int64, annoy_index_search_k_nodes, -1) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_usearch_index, false) \ - MAKE_OBSOLETE(M, Bool, optimize_move_functions_out_of_any, false) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_undrop_table_query, true) \ - MAKE_OBSOLETE(M, Bool, allow_experimental_s3queue, true) \ - MAKE_OBSOLETE(M, Bool, query_plan_optimize_primary_key, true) \ - MAKE_OBSOLETE(M, Bool, optimize_monotonous_functions_in_order_by, false) \ - MAKE_OBSOLETE(M, UInt64, http_max_chunk_size, 100_GiB) \ - - /** The section above is for obsolete settings. Do not add anything there. */ +/// List of available types supported in Settings object (!= MergeTreeSettings, MySQLSettings, etc) +#define COMMON_SETTINGS_SUPPORTED_TYPES(CLASS_NAME, M) \ + M(CLASS_NAME, ArrowCompression) \ + M(CLASS_NAME, Bool) \ + M(CLASS_NAME, CapnProtoEnumComparingMode) \ + M(CLASS_NAME, Char) \ + M(CLASS_NAME, DateTimeInputFormat) \ + M(CLASS_NAME, DateTimeOutputFormat) \ + M(CLASS_NAME, DateTimeOverflowBehavior) \ + M(CLASS_NAME, DefaultDatabaseEngine) \ + M(CLASS_NAME, DefaultTableEngine) \ + M(CLASS_NAME, Dialect) \ + M(CLASS_NAME, DistributedDDLOutputMode) \ + M(CLASS_NAME, DistributedProductMode) \ + M(CLASS_NAME, Double) \ + M(CLASS_NAME, EscapingRule) \ + M(CLASS_NAME, Float) \ + M(CLASS_NAME, IdentifierQuotingStyle) \ + M(CLASS_NAME, Int32) \ + M(CLASS_NAME, Int64) \ + M(CLASS_NAME, IntervalOutputFormat) \ + M(CLASS_NAME, JoinAlgorithm) \ + M(CLASS_NAME, JoinStrictness) \ + M(CLASS_NAME, LoadBalancing) \ + M(CLASS_NAME, LocalFSReadMethod) \ + M(CLASS_NAME, LogQueriesType) \ + M(CLASS_NAME, LogsLevel) \ + M(CLASS_NAME, Map) \ + M(CLASS_NAME, MaxThreads) \ + M(CLASS_NAME, Milliseconds) \ + M(CLASS_NAME, MsgPackUUIDRepresentation) \ + M(CLASS_NAME, MySQLDataTypesSupport) \ + M(CLASS_NAME, NonZeroUInt64) \ + M(CLASS_NAME, ORCCompression) \ + M(CLASS_NAME, OverflowMode) \ + M(CLASS_NAME, OverflowModeGroupBy) \ + M(CLASS_NAME, ParallelReplicasCustomKeyFilterType) \ + M(CLASS_NAME, ParquetCompression) \ + M(CLASS_NAME, ParquetVersion) \ + M(CLASS_NAME, QueryCacheNondeterministicFunctionHandling) \ + M(CLASS_NAME, QueryCacheSystemTableHandling) \ + M(CLASS_NAME, SchemaInferenceMode) \ + M(CLASS_NAME, Seconds) \ + M(CLASS_NAME, SetOperationMode) \ + M(CLASS_NAME, ShortCircuitFunctionEvaluation) \ + M(CLASS_NAME, SQLSecurityType) \ + M(CLASS_NAME, StreamingHandleErrorMode) \ + M(CLASS_NAME, String) \ + M(CLASS_NAME, Timezone) \ + M(CLASS_NAME, TotalsMode) \ + M(CLASS_NAME, TransactionsWaitCSNMode) \ + M(CLASS_NAME, UInt64) \ + M(CLASS_NAME, UInt64Auto) \ + M(CLASS_NAME, URI) -#define FORMAT_FACTORY_SETTINGS(M, ALIAS) \ - M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \ - M(Bool, format_csv_allow_single_quotes, false, "If it is set to true, allow strings in single quotes.", 0) \ - M(Bool, format_csv_allow_double_quotes, true, "If it is set to true, allow strings in double quotes.", 0) \ - M(Bool, output_format_csv_serialize_tuple_into_separate_columns, true, "If it set to true, then Tuples in CSV format are serialized as separate columns (that is, their nesting in the tuple is lost)", 0) \ - M(Bool, input_format_csv_deserialize_separate_columns_into_tuple, true, "If it set to true, then separate columns written in CSV format can be deserialized to Tuple column.", 0) \ - M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \ - M(Bool, input_format_csv_allow_cr_end_of_line, false, "If it is set true, \\r will be allowed at end of line not followed by \\n", 0) \ - M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices", 0) \ - M(Bool, input_format_csv_arrays_as_nested_csv, false, R"(When reading Array from CSV, expect that its elements were serialized in nested CSV and then put into string. Example: "[""Hello"", ""world"", ""42"""" TV""]". Braces around array can be omitted.)", 0) \ - M(Bool, input_format_skip_unknown_fields, true, "Skip columns with unknown names from input data (it works for JSONEachRow, -WithNames, -WithNamesAndTypes and TSKV formats).", 0) \ - M(Bool, input_format_with_names_use_header, true, "For -WithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ - M(Bool, input_format_with_types_use_header, true, "For -WithNamesAndTypes input formats this controls whether format parser should check if data types from the input match data types from the header.", 0) \ - M(Bool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ - M(Bool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, -WithNames, -WithNamesAndTypes formats).", IMPORTANT) \ - M(Bool, input_format_csv_empty_as_default, true, "Treat empty fields in CSV input as default values.", 0) \ - M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ - M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices.", 0) \ - M(Bool, input_format_null_as_default, true, "Initialize null fields with default values if the data type of this field is not nullable and it is supported by the input format", 0) \ - M(Bool, input_format_force_null_for_omitted_fields, false, "Force initialize omitted fields with null values", 0) \ - M(Bool, input_format_arrow_case_insensitive_column_matching, false, "Ignore case when matching Arrow columns with CH columns.", 0) \ - M(Int64, input_format_orc_row_batch_size, 100'000, "Batch size when reading ORC stripes.", 0) \ - M(Bool, input_format_orc_case_insensitive_column_matching, false, "Ignore case when matching ORC columns with CH columns.", 0) \ - M(Bool, input_format_parquet_case_insensitive_column_matching, false, "Ignore case when matching Parquet columns with CH columns.", 0) \ - M(Bool, input_format_parquet_preserve_order, false, "Avoid reordering rows when reading from Parquet files. Usually makes it much slower.", 0) \ - M(Bool, input_format_parquet_filter_push_down, true, "When reading Parquet files, skip whole row groups based on the WHERE/PREWHERE expressions and min/max statistics in the Parquet metadata.", 0) \ - M(Bool, input_format_parquet_use_native_reader, false, "When reading Parquet files, to use native reader instead of arrow reader.", 0) \ - M(Bool, input_format_allow_seeks, true, "Allow seeks while reading in ORC/Parquet/Arrow input formats", 0) \ - M(Bool, input_format_orc_allow_missing_columns, true, "Allow missing columns while reading ORC input formats", 0) \ - M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \ - M(Bool, input_format_orc_filter_push_down, true, "When reading ORC files, skip whole stripes or row groups based on the WHERE/PREWHERE expressions, min/max statistics or bloom filter in the ORC metadata.", 0) \ - M(String, input_format_orc_reader_time_zone_name, "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT.", 0) \ - M(Bool, input_format_parquet_allow_missing_columns, true, "Allow missing columns while reading Parquet input formats", 0) \ - M(UInt64, input_format_parquet_local_file_min_bytes_for_seek, 8192, "Min bytes required for local read (file) to do seek, instead of read with ignore in Parquet input format", 0) \ - M(Bool, input_format_arrow_allow_missing_columns, true, "Allow missing columns while reading Arrow input formats", 0) \ - M(Char, input_format_hive_text_fields_delimiter, '\x01', "Delimiter between fields in Hive Text File", 0) \ - M(Char, input_format_hive_text_collection_items_delimiter, '\x02', "Delimiter between collection(array or map) items in Hive Text File", 0) \ - M(Char, input_format_hive_text_map_keys_delimiter, '\x03', "Delimiter between a pair of map key/values in Hive Text File", 0) \ - M(Bool, input_format_hive_text_allow_variable_number_of_columns, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values", 0) \ - M(UInt64, input_format_msgpack_number_of_columns, 0, "The number of columns in inserted MsgPack data. Used for automatic schema inference from data.", 0) \ - M(MsgPackUUIDRepresentation, output_format_msgpack_uuid_representation, FormatSettings::MsgPackUUIDRepresentation::EXT, "The way how to output UUID in MsgPack format.", 0) \ - M(UInt64, input_format_max_rows_to_read_for_schema_inference, 25000, "The maximum rows of data to read for automatic schema inference", 0) \ - M(UInt64, input_format_max_bytes_to_read_for_schema_inference, 32 * 1024 * 1024, "The maximum bytes of data to read for automatic schema inference", 0) \ - M(Bool, input_format_csv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in CSV format", 0) \ - M(Bool, input_format_csv_try_infer_numbers_from_strings, false, "Try to infer numbers from string fields while schema inference in CSV format", 0) \ - M(Bool, input_format_csv_try_infer_strings_from_quoted_tuples, true, "Interpret quoted tuples in the input data as a value of type String.", 0) \ - M(Bool, input_format_tsv_use_best_effort_in_schema_inference, true, "Use some tweaks and heuristics to infer schema in TSV format", 0) \ - M(Bool, input_format_csv_detect_header, true, "Automatically detect header with names and types in CSV format", 0) \ - M(Bool, input_format_csv_allow_whitespace_or_tab_as_delimiter, false, "Allow to use spaces and tabs(\\t) as field delimiter in the CSV strings", 0) \ - M(Bool, input_format_csv_trim_whitespaces, true, "Trims spaces and tabs (\\t) characters at the beginning and end in CSV strings", 0) \ - M(Bool, input_format_csv_use_default_on_bad_values, false, "Allow to set default value to column when CSV field deserialization failed on bad value", 0) \ - M(Bool, input_format_csv_allow_variable_number_of_columns, false, "Ignore extra columns in CSV input (if file has more columns than expected) and treat missing fields in CSV input as default values", 0) \ - M(Bool, input_format_tsv_allow_variable_number_of_columns, false, "Ignore extra columns in TSV input (if file has more columns than expected) and treat missing fields in TSV input as default values", 0) \ - M(Bool, input_format_custom_allow_variable_number_of_columns, false, "Ignore extra columns in CustomSeparated input (if file has more columns than expected) and treat missing fields in CustomSeparated input as default values", 0) \ - M(Bool, input_format_json_compact_allow_variable_number_of_columns, false, "Ignore extra columns in JSONCompact(EachRow) input (if file has more columns than expected) and treat missing fields in JSONCompact(EachRow) input as default values", 0) \ - M(Bool, input_format_tsv_detect_header, true, "Automatically detect header with names and types in TSV format", 0) \ - M(Bool, input_format_custom_detect_header, true, "Automatically detect header with names and types in CustomSeparated format", 0) \ - M(Bool, input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format Parquet", 0) \ - M(UInt64, input_format_parquet_max_block_size, DEFAULT_BLOCK_SIZE, "Max block size for parquet reader.", 0) \ - M(UInt64, input_format_parquet_prefer_block_bytes, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader", 0) \ - M(Bool, input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format Protobuf", 0) \ - M(Bool, input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format CapnProto", 0) \ - M(Bool, input_format_orc_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format ORC", 0) \ - M(Bool, input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference, false, "Skip columns with unsupported types while schema inference for format Arrow", 0) \ - M(String, column_names_for_schema_inference, "", "The list of column names to use in schema inference for formats without column names. The format: 'column1,column2,column3,...'", 0) \ - M(String, schema_inference_hints, "", "The list of column names and types to use in schema inference for formats without column names. The format: 'column_name1 column_type1, column_name2 column_type2, ...'", 0) \ - M(SchemaInferenceMode, schema_inference_mode, "default", "Mode of schema inference. 'default' - assume that all files have the same schema and schema can be inferred from any file, 'union' - files can have different schemas and the resulting schema should be the a union of schemas of all files", 0) \ - M(UInt64Auto, schema_inference_make_columns_nullable, 1, "If set to true, all inferred types will be Nullable in schema inference. When set to false, no columns will be converted to Nullable. When set to 'auto', ClickHouse will use information about nullability from the data.", 0) \ - M(Bool, input_format_json_read_bools_as_numbers, true, "Allow to parse bools as numbers in JSON input formats", 0) \ - M(Bool, input_format_json_read_bools_as_strings, true, "Allow to parse bools as strings in JSON input formats", 0) \ - M(Bool, input_format_json_try_infer_numbers_from_strings, false, "Try to infer numbers from string fields while schema inference", 0) \ - M(Bool, input_format_json_validate_types_from_metadata, true, "For JSON/JSONCompact/JSONColumnsWithMetadata input formats this controls whether format parser should check if data types from input metadata match data types of the corresponding columns from the table", 0) \ - M(Bool, input_format_json_read_numbers_as_strings, true, "Allow to parse numbers as strings in JSON input formats", 0) \ - M(Bool, input_format_json_read_objects_as_strings, true, "Allow to parse JSON objects as strings in JSON input formats", 0) \ - M(Bool, input_format_json_read_arrays_as_strings, true, "Allow to parse JSON arrays as strings in JSON input formats", 0) \ - M(Bool, input_format_json_try_infer_named_tuples_from_objects, true, "Try to infer named tuples from JSON objects in JSON input formats", 0) \ - M(Bool, input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects, false, "Use String type instead of an exception in case of ambiguous paths in JSON objects during named tuples inference", 0) \ - M(Bool, input_format_json_infer_incomplete_types_as_strings, true, "Use type String for keys that contains only Nulls or empty objects/arrays during schema inference in JSON input formats", 0) \ - M(Bool, input_format_json_named_tuples_as_objects, true, "Deserialize named tuple columns as JSON objects", 0) \ - M(Bool, input_format_json_ignore_unknown_keys_in_named_tuple, true, "Ignore unknown keys in json object for named tuples", 0) \ - M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \ - M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \ - M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \ - M(Bool, input_format_try_infer_variants, false, "Try to infer the Variant type in text formats when there is more than one possible type for column/array elements", 0) \ - M(Bool, type_json_skip_duplicated_paths, false, "When enabled, during parsing JSON object into JSON type duplicated paths will be ignored and only the first one will be inserted instead of an exception", 0) \ - M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \ - M(Bool, input_format_json_empty_as_default, false, "Treat empty fields in JSON input as default values.", 0) \ - M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \ - M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \ - M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \ - M(Bool, input_format_try_infer_datetimes_only_datetime64, false, "When input_format_try_infer_datetimes is enabled, infer only DateTime64 but not DateTime types", 0) \ - M(Bool, input_format_try_infer_exponent_floats, false, "Try to infer floats in exponential notation while schema inference in text formats (except JSON, where exponent numbers are always inferred)", 0) \ - M(Bool, output_format_markdown_escape_special_characters, false, "Escape special characters in Markdown", 0) \ - M(Bool, input_format_protobuf_flatten_google_wrappers, false, "Enable Google wrappers for regular non-nested columns, e.g. google.protobuf.StringValue 'str' for String column 'str'. For Nullable columns empty wrappers are recognized as defaults, and missing as nulls", 0) \ - M(Bool, output_format_protobuf_nullables_with_google_wrappers, false, "When serializing Nullable columns with Google wrappers, serialize default values as empty wrappers. If turned off, default and null values are not serialized", 0) \ - M(UInt64, input_format_csv_skip_first_lines, 0, "Skip specified number of lines at the beginning of data in CSV format", 0) \ - M(UInt64, input_format_tsv_skip_first_lines, 0, "Skip specified number of lines at the beginning of data in TSV format", 0) \ - M(Bool, input_format_csv_skip_trailing_empty_lines, false, "Skip trailing empty lines in CSV format", 0) \ - M(Bool, input_format_tsv_skip_trailing_empty_lines, false, "Skip trailing empty lines in TSV format", 0) \ - M(Bool, input_format_custom_skip_trailing_empty_lines, false, "Skip trailing empty lines in CustomSeparated format", 0) \ - M(Bool, input_format_tsv_crlf_end_of_line, false, "If it is set true, file function will read TSV format with \\r\\n instead of \\n.", 0) \ - \ - M(Bool, input_format_native_allow_types_conversion, true, "Allow data types conversion in Native input format", 0) \ - M(Bool, input_format_native_decode_types_in_binary_format, false, "Read data types in binary format instead of type names in Native input format", 0) \ - M(Bool, output_format_native_encode_types_in_binary_format, false, "Write data types in binary format instead of type names in Native output format", 0) \ - \ - M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \ - M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \ - M(IntervalOutputFormat, interval_output_format, FormatSettings::IntervalOutputFormat::Numeric, "Textual representation of Interval. Possible values: 'kusto', 'numeric'.", 0) \ - \ - M(Bool, input_format_ipv4_default_on_conversion_error, false, "Deserialization of IPv4 will use default values instead of throwing exception on conversion error.", 0) \ - M(Bool, input_format_ipv6_default_on_conversion_error, false, "Deserialization of IPV6 will use default values instead of throwing exception on conversion error.", 0) \ - M(String, bool_true_representation, "true", "Text to represent bool value in TSV/CSV formats.", 0) \ - M(String, bool_false_representation, "false", "Text to represent bool value in TSV/CSV formats.", 0) \ - \ - M(Bool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \ - M(Bool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \ - M(Bool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \ - M(Bool, input_format_avro_allow_missing_fields, false, "For Avro/AvroConfluent format: when field is not found in schema use default value instead of error", 0) \ - /** This setting is obsolete and do nothing, left for compatibility reasons. */ \ - M(Bool, input_format_avro_null_as_default, false, "For Avro/AvroConfluent format: insert default in case of null and non Nullable column", 0) \ - M(UInt64, format_binary_max_string_size, 1_GiB, "The maximum allowed size for String in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \ - M(UInt64, format_binary_max_array_size, 1_GiB, "The maximum allowed size for Array in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \ - M(Bool, input_format_binary_decode_types_in_binary_format, false, "Read data types in binary format instead of type names in RowBinaryWithNamesAndTypes input format", 0) \ - M(Bool, output_format_binary_encode_types_in_binary_format, false, "Write data types in binary format instead of type names in RowBinaryWithNamesAndTypes output format ", 0) \ - M(URI, format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \ - \ - M(Bool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \ - M(Bool, output_format_json_quote_denormals, false, "Enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format.", 0) \ - M(Bool, output_format_json_quote_decimals, false, "Controls quoting of decimals in JSON output format.", 0) \ - M(Bool, output_format_json_quote_64bit_floats, false, "Controls quoting of 64-bit float numbers in JSON output format.", 0) \ - \ - M(Bool, output_format_json_escape_forward_slashes, true, "Controls escaping forward slashes for string outputs in JSON output format. This is intended for compatibility with JavaScript. Don't confuse with backslashes that are always escaped.", 0) \ - M(Bool, output_format_json_named_tuples_as_objects, true, "Serialize named tuple columns as JSON objects.", 0) \ - M(Bool, output_format_json_skip_null_value_in_named_tuples, false, "Skip key value pairs with null value when serialize named tuple columns as JSON objects. It is only valid when output_format_json_named_tuples_as_objects is true.", 0) \ - M(Bool, output_format_json_array_of_rows, false, "Output a JSON array of all rows in JSONEachRow(Compact) format.", 0) \ - M(Bool, output_format_json_validate_utf8, false, "Validate UTF-8 sequences in JSON output formats, doesn't impact formats JSON/JSONCompact/JSONColumnsWithMetadata, they always validate utf8", 0) \ - \ - M(String, format_json_object_each_row_column_for_object_name, "", "The name of column that will be used as object names in JSONObjectEachRow format. Column type should be String", 0) \ - \ - M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ - M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ - M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \ - M(UInt64, output_format_pretty_max_value_width_apply_for_single_value, false, "Only cut values (see the `output_format_pretty_max_value_width` setting) when it is not a single value in a block. Otherwise output it entirely, which is useful for the `SHOW CREATE TABLE` query.", 0) \ - M(UInt64Auto, output_format_pretty_color, "auto", "Use ANSI escape sequences in Pretty formats. 0 - disabled, 1 - enabled, 'auto' - enabled if a terminal.", 0) \ - M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \ - M(UInt64, output_format_pretty_display_footer_column_names, true, "Display column names in the footer if there are 999 or more rows.", 0) \ - M(UInt64, output_format_pretty_display_footer_column_names_min_rows, 50, "Sets the minimum threshold value of rows for which to enable displaying column names in the footer. 50 (default)", 0) \ - M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \ - M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \ - M(Bool, output_format_parquet_string_as_string, true, "Use Parquet String type instead of Binary for String columns.", 0) \ - M(Bool, output_format_parquet_fixed_string_as_fixed_byte_array, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary for FixedString columns.", 0) \ - M(ParquetVersion, output_format_parquet_version, "2.latest", "Parquet format version for output format. Supported versions: 1.0, 2.4, 2.6 and 2.latest (default)", 0) \ - M(ParquetCompression, output_format_parquet_compression_method, "zstd", "Compression method for Parquet output format. Supported codecs: snappy, lz4, brotli, zstd, gzip, none (uncompressed)", 0) \ - M(Bool, output_format_parquet_compliant_nested_types, true, "In parquet file schema, use name 'element' instead of 'item' for list elements. This is a historical artifact of Arrow library implementation. Generally increases compatibility, except perhaps with some old versions of Arrow.", 0) \ - M(Bool, output_format_parquet_use_custom_encoder, true, "Use a faster Parquet encoder implementation.", 0) \ - M(Bool, output_format_parquet_parallel_encoding, true, "Do Parquet encoding in multiple threads. Requires output_format_parquet_use_custom_encoder.", 0) \ - M(UInt64, output_format_parquet_data_page_size, 1024 * 1024, "Target page size in bytes, before compression.", 0) \ - M(UInt64, output_format_parquet_batch_size, 1024, "Check page size every this many rows. Consider decreasing if you have columns with average values size above a few KBs.", 0) \ - M(Bool, output_format_parquet_write_page_index, true, "Add a possibility to write page index into parquet files.", 0) \ - M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy', 'zstd'.", 0) \ - M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ - M(String, output_format_avro_string_column_pattern, "", "For Avro format: regexp of String columns to select as AVRO string.", 0) \ - M(UInt64, output_format_avro_rows_in_file, 1, "Max rows in a file (if permitted by storage)", 0) \ - M(Bool, output_format_tsv_crlf_end_of_line, false, "If it is set true, end of line in TSV format will be \\r\\n instead of \\n.", 0) \ - M(String, format_csv_null_representation, "\\N", "Custom NULL representation in CSV format", 0) \ - M(String, format_tsv_null_representation, "\\N", "Custom NULL representation in TSV format", 0) \ - M(Bool, output_format_decimal_trailing_zeros, false, "Output trailing zeros when printing Decimal values. E.g. 1.230000 instead of 1.23.", 0) \ - \ - M(UInt64, input_format_allow_errors_num, 0, "Maximum absolute amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ - M(Float, input_format_allow_errors_ratio, 0, "Maximum relative amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ - M(String, input_format_record_errors_file_path, "", "Path of the file used to record errors while reading text formats (CSV, TSV).", 0) \ - M(String, errors_output_format, "CSV", "Method to write Errors to text output.", 0) \ - \ - M(String, format_schema, "", "Schema identifier (used by schema-based formats)", 0) \ - M(String, format_template_resultset, "", "Path to file which contains format string for result set (for Template format)", 0) \ - M(String, format_template_row, "", "Path to file which contains format string for rows (for Template format)", 0) \ - M(String, format_template_row_format, "", "Format string for rows (for Template format)", 0) \ - M(String, format_template_resultset_format, "", "Format string for result set (for Template format)", 0) \ - M(String, format_template_rows_between_delimiter, "\n", "Delimiter between rows (for Template format)", 0) \ - \ - M(EscapingRule, format_custom_escaping_rule, "Escaped", "Field escaping rule (for CustomSeparated format)", 0) \ - M(String, format_custom_field_delimiter, "\t", "Delimiter between fields (for CustomSeparated format)", 0) \ - M(String, format_custom_row_before_delimiter, "", "Delimiter before field of the first column (for CustomSeparated format)", 0) \ - M(String, format_custom_row_after_delimiter, "\n", "Delimiter after field of the last column (for CustomSeparated format)", 0) \ - M(String, format_custom_row_between_delimiter, "", "Delimiter between rows (for CustomSeparated format)", 0) \ - M(String, format_custom_result_before_delimiter, "", "Prefix before result set (for CustomSeparated format)", 0) \ - M(String, format_custom_result_after_delimiter, "", "Suffix after result set (for CustomSeparated format)", 0) \ - \ - M(String, format_regexp, "", "Regular expression (for Regexp format)", 0) \ - M(EscapingRule, format_regexp_escaping_rule, "Raw", "Field escaping rule (for Regexp format)", 0) \ - M(Bool, format_regexp_skip_unmatched, false, "Skip lines unmatched by regular expression (for Regexp format)", 0) \ - \ - M(Bool, output_format_enable_streaming, false, "Enable streaming in output formats that support it.", 0) \ - M(Bool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ - M(Bool, output_format_pretty_row_numbers, true, "Add row numbers before each row for pretty output format", 0) \ - M(Bool, output_format_pretty_highlight_digit_groups, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline.", 0) \ - M(UInt64, output_format_pretty_single_large_number_tip_threshold, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)", 0) \ - M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \ - \ - M(Bool, exact_rows_before_limit, false, "When enabled, ClickHouse will provide exact value for rows_before_limit_at_least statistic, but with the cost that the data before limit will have to be read completely", 0) \ - M(Bool, rows_before_aggregation, false, "When enabled, ClickHouse will provide exact value for rows_before_aggregation statistic, represents the number of rows read before aggregation", 0) \ - M(UInt64, cross_to_inner_join_rewrite, 1, "Use inner join instead of comma/cross join if there are joining expressions in the WHERE section. Values: 0 - no rewrite, 1 - apply if possible for comma/cross, 2 - force rewrite all comma joins, cross - if possible", 0) \ - \ - M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \ - M(Bool, output_format_arrow_use_signed_indexes_for_dictionary, true, "Use signed integers for dictionary indexes in Arrow format", 0) \ - M(Bool, output_format_arrow_use_64_bit_indexes_for_dictionary, false, "Always use 64 bit integers for dictionary indexes in Arrow format", 0) \ - M(Bool, output_format_arrow_string_as_string, true, "Use Arrow String type instead of Binary for String columns", 0) \ - M(Bool, output_format_arrow_fixed_string_as_fixed_byte_array, true, "Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns.", 0) \ - M(ArrowCompression, output_format_arrow_compression_method, "lz4_frame", "Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed)", 0) \ - \ - M(Bool, output_format_orc_string_as_string, true, "Use ORC String type instead of Binary for String columns", 0) \ - M(ORCCompression, output_format_orc_compression_method, "zstd", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \ - M(UInt64, output_format_orc_row_index_stride, 10'000, "Target row index stride in ORC output format", 0) \ - M(Double, output_format_orc_dictionary_key_size_threshold, 0.0, "For a string column in ORC output format, if the number of distinct values is greater than this fraction of the total number of non-null rows, turn off dictionary encoding. Otherwise dictionary encoding is enabled", 0) \ - \ - M(CapnProtoEnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::CapnProtoEnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \ - \ - M(Bool, format_capn_proto_use_autogenerated_schema, true, "Use autogenerated CapnProto schema when format_schema is not set", 0) \ - M(Bool, format_protobuf_use_autogenerated_schema, true, "Use autogenerated Protobuf when format_schema is not set", 0) \ - M(String, output_format_schema, "", "The path to the file where the automatically generated schema will be saved", 0) \ - \ - M(String, input_format_mysql_dump_table_name, "", "Name of the table in MySQL dump from which to read data", 0) \ - M(Bool, input_format_mysql_dump_map_column_names, true, "Match columns from table in MySQL dump and columns from ClickHouse table by names", 0) \ - \ - M(UInt64, output_format_sql_insert_max_batch_size, DEFAULT_BLOCK_SIZE, "The maximum number of rows in one INSERT statement.", 0) \ - M(String, output_format_sql_insert_table_name, "table", "The name of table in the output INSERT query", 0) \ - M(Bool, output_format_sql_insert_include_column_names, true, "Include column names in INSERT query", 0) \ - M(Bool, output_format_sql_insert_use_replace, false, "Use REPLACE statement instead of INSERT", 0) \ - M(Bool, output_format_sql_insert_quote_names, true, "Quote column names with '`' characters", 0) \ - \ - M(Bool, output_format_values_escape_quote_with_quote, false, "If true escape ' with '', otherwise quoted with \\'", 0) \ - \ - M(Bool, output_format_bson_string_as_string, false, "Use BSON String type instead of Binary for String columns.", 0) \ - M(Bool, input_format_bson_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format BSON.", 0) \ - \ - M(Bool, format_display_secrets_in_show_and_select, false, "Do not hide secrets in SHOW and SELECT queries.", IMPORTANT) \ - M(Bool, regexp_dict_allow_hyperscan, true, "Allow regexp_tree dictionary using Hyperscan library.", 0) \ - M(Bool, regexp_dict_flag_case_insensitive, false, "Use case-insensitive matching for a regexp_tree dictionary. Can be overridden in individual expressions with (?i) and (?-i).", 0) \ - M(Bool, regexp_dict_flag_dotall, false, "Allow '.' to match newline characters for a regexp_tree dictionary.", 0) \ - \ - M(Bool, dictionary_use_async_executor, false, "Execute a pipeline for reading dictionary source in several threads. It's supported only by dictionaries with local CLICKHOUSE source.", 0) \ - M(Bool, precise_float_parsing, false, "Prefer more precise (but slower) float parsing algorithm", 0) \ - M(DateTimeOverflowBehavior, date_time_overflow_behavior, "ignore", "Overflow mode for Date, Date32, DateTime, DateTime64 types. Possible values: 'ignore', 'throw', 'saturate'.", 0) \ - M(Bool, validate_experimental_and_suspicious_types_inside_nested_types, true, "Validate usage of experimental and suspicious types inside nested types like Array/Map/Tuple", 0) \ - \ - M(Bool, output_format_always_quote_identifiers, false, "Always quote identifiers", 0) \ - M(IdentifierQuotingStyle, output_format_identifier_quoting_style, IdentifierQuotingStyle::Backticks, "Set the quoting style for identifiers", 0) \ - - -// End of FORMAT_FACTORY_SETTINGS -// Please add settings non-related to formats into the COMMON_SETTINGS above. - -#define OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \ - /** Obsolete format settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ - MAKE_OBSOLETE(M, Bool, input_format_arrow_import_nested, false) \ - MAKE_OBSOLETE(M, Bool, input_format_parquet_import_nested, false) \ - MAKE_OBSOLETE(M, Bool, input_format_orc_import_nested, false) \ - -#define LIST_OF_SETTINGS(M, ALIAS) \ - COMMON_SETTINGS(M, ALIAS) \ - OBSOLETE_SETTINGS(M, ALIAS) \ - FORMAT_FACTORY_SETTINGS(M, ALIAS) \ - OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \ - -DECLARE_SETTINGS_TRAITS_ALLOW_CUSTOM_SETTINGS(SettingsTraits, LIST_OF_SETTINGS) - - -/** Settings of query execution. - * These settings go to users.xml. - */ -struct Settings : public BaseSettings, public IHints<2> +COMMON_SETTINGS_SUPPORTED_TYPES(Settings, DECLARE_SETTING_TRAIT) +struct Settings { - Settings() = default; + Settings(); + Settings(const Settings & settings); + Settings(Settings && settings) noexcept; + ~Settings(); - /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings). - * The profile can also be set using the `set` functions, like the profile setting. - */ - void setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config); + Settings & operator=(const Settings & other); + bool operator==(const Settings & other) const; - /// Load settings from configuration file, at "path" prefix in configuration. - void loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config); + COMMON_SETTINGS_SUPPORTED_TYPES(Settings, DECLARE_SETTING_SUBSCRIPT_OPERATOR) - /// Dumps profile events to column of type Map(String, String) - void dumpToMapColumn(IColumn * column, bool changed_only = true); + /// General API as needed + bool has(std::string_view name) const; + bool isChanged(std::string_view name) const; - /// Check that there is no user-level settings at the top level in config. - /// This is a common source of mistake (user don't know where to write user-level setting). + bool tryGet(std::string_view name, Field & value) const; + Field get(std::string_view name) const; + + void set(std::string_view name, const Field & value); + void setDefaultValue(std::string_view name); + + std::vector getHints(const String & name) const; + String toString() const; + + SettingsChanges changes() const; + void applyChanges(const SettingsChanges & changes); + std::vector getAllRegisteredNames() const; + std::vector getChangedAndObsoleteNames() const; + std::vector getUnchangedNames() const; + + void dumpToSystemSettingsColumns(MutableColumnsAndConstraints & params) const; + void dumpToMapColumn(IColumn * column, bool changed_only = true) const; + NameToNameMap toNameToNameMap() const; + + void write(WriteBuffer & out, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT) const; + void read(ReadBuffer & in, SettingsWriteFormat format = SettingsWriteFormat::DEFAULT); + + void addToProgramOptions(boost::program_options::options_description & options); + void addToProgramOptions(std::string_view setting_name, boost::program_options::options_description & options); + void addToProgramOptionsAsMultitokens(boost::program_options::options_description & options) const; + void addToClientOptions( + Poco::Util::LayeredConfiguration & config, const boost::program_options::variables_map & options, bool repeated_settings) const; + + static Field castValueUtil(std::string_view name, const Field & value); + static String valueToStringUtil(std::string_view name, const Field & value); + static Field stringToValueUtil(std::string_view name, const String & str); + static bool hasBuiltin(std::string_view name); + static std::string_view resolveName(std::string_view name); static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path); - std::vector getAllRegisteredNames() const override; - - void set(std::string_view name, const Field & value) override; - - void setDefaultValue(const String & name) { resetToDefault(name); } - private: - void applyCompatibilitySetting(const String & compatibility); - - std::unordered_set settings_changed_by_compatibility_setting; + std::unique_ptr impl; }; - -#define LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS) \ - FORMAT_FACTORY_SETTINGS(M, ALIAS) \ - OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \ - -/* - * User-specified file format settings for File and URL engines. - */ -DECLARE_SETTINGS_TRAITS(FormatFactorySettingsTraits, LIST_OF_ALL_FORMAT_SETTINGS) - -struct FormatFactorySettings : public BaseSettings -{ -}; - } - -#endif diff --git a/src/Core/SettingsObsoleteMacros.h b/src/Core/SettingsObsoleteMacros.h new file mode 100644 index 00000000000..97db1def294 --- /dev/null +++ b/src/Core/SettingsObsoleteMacros.h @@ -0,0 +1,9 @@ +#pragma once + +// clang-format off +#define MAKE_OBSOLETE(M, TYPE, NAME, DEFAULT) \ + M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE) + +/// NOTE: ServerSettings::loadSettingsFromConfig() should be updated to include this settings +#define MAKE_DEPRECATED_BY_SERVER_CONFIG(M, TYPE, NAME, DEFAULT) \ + M(TYPE, NAME, DEFAULT, "User-level setting is deprecated, and it must be defined in the server configuration instead.", BaseSettingsHelpers::Flags::OBSOLETE) diff --git a/src/Core/SettingsQuirks.cpp b/src/Core/SettingsQuirks.cpp index 3127a5ef36d..47e786409b3 100644 --- a/src/Core/SettingsQuirks.cpp +++ b/src/Core/SettingsQuirks.cpp @@ -1,6 +1,6 @@ +#include #include #include -#include #include #include #include @@ -48,26 +48,35 @@ bool queryProfilerWorks() { return false; } namespace DB { +namespace Setting +{ + extern const SettingsBool async_query_sending_for_remote; + extern const SettingsBool async_socket_for_remote; + extern const SettingsUInt64 query_profiler_cpu_time_period_ns; + extern const SettingsUInt64 query_profiler_real_time_period_ns; + extern const SettingsBool use_hedged_requests; +} + /// Update some settings defaults to avoid some known issues. void applySettingsQuirks(Settings & settings, LoggerPtr log) { if (!nestedEpollWorks(log)) { - if (!settings.async_socket_for_remote.changed && settings.async_socket_for_remote) + if (!settings[Setting::async_socket_for_remote].changed && settings[Setting::async_socket_for_remote]) { - settings.async_socket_for_remote = false; + settings[Setting::async_socket_for_remote] = false; if (log) LOG_WARNING(log, "async_socket_for_remote has been disabled (you can explicitly enable it still)"); } - if (!settings.async_query_sending_for_remote.changed && settings.async_query_sending_for_remote) + if (!settings[Setting::async_query_sending_for_remote].changed && settings[Setting::async_query_sending_for_remote]) { - settings.async_query_sending_for_remote = false; + settings[Setting::async_query_sending_for_remote] = false; if (log) LOG_WARNING(log, "async_query_sending_for_remote has been disabled (you can explicitly enable it still)"); } - if (!settings.use_hedged_requests.changed && settings.use_hedged_requests) + if (!settings[Setting::use_hedged_requests].changed && settings[Setting::use_hedged_requests]) { - settings.use_hedged_requests = false; + settings[Setting::use_hedged_requests] = false; if (log) LOG_WARNING(log, "use_hedged_requests has been disabled (you can explicitly enable it still)"); } @@ -75,15 +84,15 @@ void applySettingsQuirks(Settings & settings, LoggerPtr log) if (!queryProfilerWorks()) { - if (settings.query_profiler_real_time_period_ns) + if (settings[Setting::query_profiler_real_time_period_ns]) { - settings.query_profiler_real_time_period_ns = 0; + settings[Setting::query_profiler_real_time_period_ns] = 0; if (log) LOG_WARNING(log, "query_profiler_real_time_period_ns has been disabled (due to server had been compiled with sanitizers)"); } - if (settings.query_profiler_cpu_time_period_ns) + if (settings[Setting::query_profiler_cpu_time_period_ns]) { - settings.query_profiler_cpu_time_period_ns = 0; + settings[Setting::query_profiler_cpu_time_period_ns] = 0; if (log) LOG_WARNING(log, "query_profiler_cpu_time_period_ns has been disabled (due to server had been compiled with sanitizers)"); } diff --git a/src/Core/SettingsWriteFormat.h b/src/Core/SettingsWriteFormat.h new file mode 100644 index 00000000000..31888771286 --- /dev/null +++ b/src/Core/SettingsWriteFormat.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +enum class SettingsWriteFormat : uint8_t +{ + BINARY = 0, /// Part of the settings are serialized as strings, and other part as variants. This is the old behaviour. + STRINGS_WITH_FLAGS = 1, /// All settings are serialized as strings. Before each value the flag `is_important` is serialized. + DEFAULT = STRINGS_WITH_FLAGS, +}; diff --git a/src/Core/examples/CMakeLists.txt b/src/Core/examples/CMakeLists.txt index 97e5e8e67e6..369c6101032 100644 --- a/src/Core/examples/CMakeLists.txt +++ b/src/Core/examples/CMakeLists.txt @@ -1,8 +1,8 @@ clickhouse_add_executable (string_pool string_pool.cpp) -target_link_libraries (string_pool PRIVATE clickhouse_common_io clickhouse_common_config ch_contrib::sparsehash) +target_link_libraries (string_pool PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config ch_contrib::sparsehash) clickhouse_add_executable (field field.cpp) -target_link_libraries (field PRIVATE dbms) +target_link_libraries (field PRIVATE dbms clickhouse_functions) clickhouse_add_executable (string_ref_hash string_ref_hash.cpp) target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io clickhouse_common_config) diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index d4d3ad58ddd..6c1dfc828fd 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -18,12 +18,13 @@ #if defined(OS_LINUX) #include #endif +#include #include #include -#include -#include #include #include +#include +#include #include #include diff --git a/src/DataTypes/DataTypeDecimalBase.cpp b/src/DataTypes/DataTypeDecimalBase.cpp index 2adfda5bb27..68bfba475d6 100644 --- a/src/DataTypes/DataTypeDecimalBase.cpp +++ b/src/DataTypes/DataTypeDecimalBase.cpp @@ -5,6 +5,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool decimal_check_overflow; +} namespace ErrorCodes { @@ -12,11 +16,11 @@ namespace ErrorCodes bool decimalCheckComparisonOverflow(ContextPtr context) { - return context->getSettingsRef().decimal_check_overflow; + return context->getSettingsRef()[Setting::decimal_check_overflow]; } bool decimalCheckArithmeticOverflow(ContextPtr context) { - return context->getSettingsRef().decimal_check_overflow; + return context->getSettingsRef()[Setting::decimal_check_overflow]; } template diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp index 107d2d48135..cff9a2e4591 100644 --- a/src/DataTypes/DataTypeFactory.cpp +++ b/src/DataTypes/DataTypeFactory.cpp @@ -17,6 +17,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -221,7 +225,7 @@ const DataTypeFactory::Value * DataTypeFactory::findCreatorByName(const String & DataTypesDictionary::const_iterator it = data_types.find(family_name); if (data_types.end() != it) { - if (query_context && query_context->getSettingsRef().log_queries) + if (query_context && query_context->getSettingsRef()[Setting::log_queries]) query_context->addQueryFactoriesInfo(Context::QueryLogFactories::DataType, family_name); return &it->second; } @@ -233,7 +237,7 @@ const DataTypeFactory::Value * DataTypeFactory::findCreatorByName(const String & DataTypesDictionary::const_iterator it = case_insensitive_data_types.find(family_name_lowercase); if (case_insensitive_data_types.end() != it) { - if (query_context && query_context->getSettingsRef().log_queries) + if (query_context && query_context->getSettingsRef()[Setting::log_queries]) query_context->addQueryFactoriesInfo(Context::QueryLogFactories::DataType, family_name_lowercase); return &it->second; } diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp index d8cbfda4afe..3a834c700df 100644 --- a/src/DataTypes/DataTypeObject.cpp +++ b/src/DataTypes/DataTypeObject.cpp @@ -32,6 +32,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_object_type; + extern const SettingsBool use_json_alias_for_old_object_type; +} namespace ErrorCodes { @@ -519,7 +524,7 @@ static DataTypePtr createJSON(const ASTPtr & arguments) if (!context) context = Context::getGlobalContextInstance(); - if (context->getSettingsRef().allow_experimental_object_type && context->getSettingsRef().use_json_alias_for_old_object_type) + if (context->getSettingsRef()[Setting::allow_experimental_object_type] && context->getSettingsRef()[Setting::use_json_alias_for_old_object_type]) { if (arguments && !arguments->children.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Experimental Object type doesn't support any arguments. If you want to use new JSON type, set settings allow_experimental_json_type = 1 and use_json_alias_for_old_object_type = 0"); diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index d149b49d465..1bee2cd23cf 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -22,6 +22,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} namespace { @@ -469,8 +475,8 @@ namespace String description = fmt::format("Query for ClickHouse dictionary {}", data.table_name); String fixed_query = removeWhereConditionPlaceholder(query); const Settings & settings = data.global_context->getSettingsRef(); - ASTPtr select = parseQuery(parser, fixed_query, description, - settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ASTPtr select = parseQuery( + parser, fixed_query, description, settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); DDLDependencyVisitor::Visitor visitor{data}; visitor.visit(select); diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index e2e2414b1ca..f93f5882693 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -23,6 +23,12 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool check_referential_table_dependencies; + extern const SettingsBool check_table_dependencies; +} + namespace ErrorCodes { extern const int UNKNOWN_TABLE; @@ -591,8 +597,8 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new waitDatabaseStarted(); - bool check_ref_deps = query_context->getSettingsRef().check_referential_table_dependencies; - bool check_loading_deps = !check_ref_deps && query_context->getSettingsRef().check_table_dependencies; + bool check_ref_deps = query_context->getSettingsRef()[Setting::check_referential_table_dependencies]; + bool check_loading_deps = !check_ref_deps && query_context->getSettingsRef()[Setting::check_table_dependencies]; if (check_ref_deps || check_loading_deps) { std::lock_guard lock(mutex); diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp index f3cfdd4fd12..f4b4a92ed2f 100644 --- a/src/Databases/DatabaseDictionary.cpp +++ b/src/Databases/DatabaseDictionary.cpp @@ -15,6 +15,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} + namespace ErrorCodes { extern const int SYNTAX_ERROR; @@ -115,8 +121,18 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, Co ParserCreateQuery parser; const char * pos = query.data(); std::string error_message; - auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, - /* hilite = */ false, "", /* allow_multi_statements = */ false, 0, settings.max_parser_depth, settings.max_parser_backtracks, true); + auto ast = tryParseQuery( + parser, + pos, + pos + query.size(), + error_message, + /* hilite = */ false, + "", + /* allow_multi_statements = */ false, + 0, + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks], + true); if (!ast && throw_on_error) throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR); @@ -135,7 +151,8 @@ ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const } const auto & settings = getContext()->getSettingsRef(); ParserCreateQuery parser; - return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks); + return parseQuery( + parser, query.data(), query.data() + query.size(), "", 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } void DatabaseDictionary::shutdown() diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index d97474bd245..b58b63861ab 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -16,6 +16,10 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -103,7 +107,7 @@ DatabasePtr DatabaseFactory::get(const ASTCreateQuery & create, const String & m DatabasePtr impl = getImpl(create, metadata_path, context); - if (impl && context->hasQueryContext() && context->getSettingsRef().log_queries) + if (impl && context->hasQueryContext() && context->getSettingsRef()[Setting::log_queries]) context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Database, impl->getEngineName()); /// Attach database metadata diff --git a/src/Databases/DatabaseFilesystem.cpp b/src/Databases/DatabaseFilesystem.cpp index 4b50e79da4a..417230e427a 100644 --- a/src/Databases/DatabaseFilesystem.cpp +++ b/src/Databases/DatabaseFilesystem.cpp @@ -22,6 +22,11 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -188,7 +193,8 @@ ASTPtr DatabaseFilesystem::getCreateDatabaseQuery() const const String query = fmt::format("CREATE DATABASE {} ENGINE = Filesystem('{}')", backQuoteIfNeed(getDatabaseName()), path); ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks); + ASTPtr ast + = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) { diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp index ceca2666e49..f072fd33bd8 100644 --- a/src/Databases/DatabaseHDFS.cpp +++ b/src/Databases/DatabaseHDFS.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,11 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -184,7 +190,8 @@ ASTPtr DatabaseHDFS::getCreateDatabaseQuery() const ParserCreateQuery parser; const String query = fmt::format("CREATE DATABASE {} ENGINE = HDFS('{}')", backQuoteIfNeed(getDatabaseName()), source); - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks); + ASTPtr ast + = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 81378fc1c64..dfade31ac1b 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -43,6 +43,14 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool force_remove_data_recursively_on_drop; + extern const SettingsBool fsync_metadata; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; @@ -251,7 +259,7 @@ void DatabaseOnDisk::createTable( WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); writeString(statement, out); out.next(); - if (settings.fsync_metadata) + if (settings[Setting::fsync_metadata]) out.sync(); out.close(); } @@ -434,7 +442,7 @@ void DatabaseOnDisk::renameTable( /// We have to lock the table before detaching, because otherwise lockExclusively will throw. But the table may not exist. bool need_lock = table != nullptr; if (need_lock) - table_lock = table->lockExclusively(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + table_lock = table->lockExclusively(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); detachTable(local_context, table_name); if (!need_lock) @@ -549,7 +557,8 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const /// If database.sql doesn't exist, then engine is Ordinary String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary"; ParserCreateQuery parser; - ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks); + ast = parseQuery( + parser, query.data(), query.data() + query.size(), "", 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) @@ -566,7 +575,7 @@ void DatabaseOnDisk::drop(ContextPtr local_context) waitDatabaseStarted(); assert(TSA_SUPPRESS_WARNING_FOR_READ(tables).empty()); - if (local_context->getSettingsRef().force_remove_data_recursively_on_drop) + if (local_context->getSettingsRef()[Setting::force_remove_data_recursively_on_drop]) { (void)fs::remove_all(std::filesystem::path(getContext()->getPath()) / data_path); (void)fs::remove_all(getMetadataPath()); @@ -691,7 +700,9 @@ void DatabaseOnDisk::iterateMetadataFiles(const IteratingFunction & process_meta process_metadata_file(file.first); else process_tmp_drop_metadata_file(file.first); - }, Priority{}, getContext()->getSettingsRef().lock_acquire_timeout.totalMicroseconds()); + }, + Priority{}, + getContext()->getSettingsRef()[Setting::lock_acquire_timeout].totalMicroseconds()); } pool.wait(); } @@ -737,8 +748,18 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata( ParserCreateQuery parser; const char * pos = query.data(); std::string error_message; - auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false, - "in file " + metadata_file_path, /* allow_multi_statements = */ false, 0, settings.max_parser_depth, settings.max_parser_backtracks, true); + auto ast = tryParseQuery( + parser, + pos, + pos + query.size(), + error_message, + /* hilite = */ false, + "in file " + metadata_file_path, + /* allow_multi_statements = */ false, + 0, + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks], + true); if (!ast && throw_on_error) throw Exception::createDeprecated(error_message, ErrorCodes::SYNTAX_ERROR); @@ -801,8 +822,8 @@ ASTPtr DatabaseOnDisk::getCreateQueryFromStorage(const String & table_name, cons storage, ast_storage, false, - static_cast(settings.max_parser_depth), - static_cast(settings.max_parser_backtracks), + static_cast(settings[Setting::max_parser_depth]), + static_cast(settings[Setting::max_parser_backtracks]), throw_on_error); create_table_query->set(create_table_query->as()->comment, @@ -854,7 +875,7 @@ void DatabaseOnDisk::modifySettingsMetadata(const SettingsChanges & settings_cha writeString(statement, out); out.next(); - if (getContext()->getSettingsRef().fsync_metadata) + if (getContext()->getSettingsRef()[Setting::fsync_metadata]) out.sync(); out.close(); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index dd8a3f42ea8..5b2f6aaeb9c 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -41,6 +41,15 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_database_ordinary; + extern const SettingsBool fsync_metadata; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsSetOperationMode union_default_mode; +} namespace ErrorCodes { @@ -171,7 +180,7 @@ void DatabaseOrdinary::convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const Qu WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); writeString(statement, out); out.next(); - if (getContext()->getSettingsRef().fsync_metadata) + if (getContext()->getSettingsRef()[Setting::fsync_metadata]) out.sync(); out.close(); } @@ -251,7 +260,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables convertMergeTreeToReplicatedIfNeeded(ast, qualified_name, file_name); - NormalizeSelectWithUnionQueryVisitor::Data data{local_context->getSettingsRef().union_default_mode}; + NormalizeSelectWithUnionQueryVisitor::Data data{local_context->getSettingsRef()[Setting::union_default_mode]}; NormalizeSelectWithUnionQueryVisitor{data}.visit(ast); std::lock_guard lock{metadata.mutex}; metadata.parsed_tables[qualified_name] = ParsedTableMetadata{full_path.string(), ast}; @@ -403,7 +412,7 @@ LoadTaskPtr DatabaseOrdinary::startupTableAsync( { /// Since startup() method can use physical paths on disk we don't allow any exclusive actions (rename, drop so on) /// until startup finished. - auto table_lock_holder = table->lockForShare(RWLockImpl::NO_QUERY, getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock_holder = table->lockForShare(RWLockImpl::NO_QUERY, getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); table->startup(); /// If table is ReplicatedMergeTree after conversion from MergeTree, @@ -550,7 +559,8 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta statement.data() + statement.size(), "in file " + table_metadata_path, 0, - local_context->getSettingsRef().max_parser_depth, local_context->getSettingsRef().max_parser_backtracks); + local_context->getSettingsRef()[Setting::max_parser_depth], + local_context->getSettingsRef()[Setting::max_parser_backtracks]); applyMetadataChangesToCreateQuery(ast, metadata); @@ -559,7 +569,7 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL); writeString(statement, out); out.next(); - if (local_context->getSettingsRef().fsync_metadata) + if (local_context->getSettingsRef()[Setting::fsync_metadata]) out.sync(); out.close(); } @@ -590,7 +600,7 @@ void registerDatabaseOrdinary(DatabaseFactory & factory) { auto create_fn = [](const DatabaseFactory::Arguments & args) { - if (!args.create_query.attach && !args.context->getSettingsRef().allow_deprecated_database_ordinary) + if (!args.create_query.attach && !args.context->getSettingsRef()[Setting::allow_deprecated_database_ordinary]) throw Exception( ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Ordinary database engine is deprecated (see also allow_deprecated_database_ordinary setting)"); diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 1bc6334db29..1f2d21d7f6a 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -48,6 +48,17 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 database_replicated_allow_replicated_engine_arguments; + extern const SettingsBool database_replicated_always_detach_permanently; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; +} + + namespace ErrorCodes { extern const int NO_ZOOKEEPER; @@ -921,7 +932,7 @@ void DatabaseReplicated::checkTableEngine(const ASTCreateQuery & query, ASTStora } /// We will replace it with default arguments if the setting is 2 - if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments != 2) + if (query_context->getSettingsRef()[Setting::database_replicated_allow_replicated_engine_arguments] != 2) throw Exception(ErrorCodes::INCORRECT_QUERY, "Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. " "If you really want to specify it explicitly, then you should use some macros " @@ -1020,7 +1031,8 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_ if (auto * query_drop = query->as()) { - if (query_drop->kind == ASTDropQuery::Kind::Detach && query_context->getSettingsRef().database_replicated_always_detach_permanently) + if (query_drop->kind == ASTDropQuery::Kind::Detach + && query_context->getSettingsRef()[Setting::database_replicated_always_detach_permanently]) query_drop->permanently = true; if (query_drop->kind == ASTDropQuery::Kind::Detach && !query_drop->permanently) throw Exception(ErrorCodes::INCORRECT_QUERY, "DETACH TABLE is not allowed for Replicated databases. " @@ -1033,7 +1045,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex { waitDatabaseStarted(); - if (query_context->getCurrentTransaction() && query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (query_context->getCurrentTransaction() && query_context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Distributed DDL queries inside transactions are not supported"); if (is_readonly) @@ -1080,9 +1092,9 @@ static UUID getTableUUIDIfReplicated(const String & metadata, ContextPtr context return UUIDHelpers::Nil; ParserCreateQuery parser; - auto size = context->getSettingsRef().max_query_size; - auto depth = context->getSettingsRef().max_parser_depth; - auto backtracks = context->getSettingsRef().max_parser_backtracks; + auto size = context->getSettingsRef()[Setting::max_query_size]; + auto depth = context->getSettingsRef()[Setting::max_parser_depth]; + auto backtracks = context->getSettingsRef()[Setting::max_parser_backtracks]; ASTPtr query = parseQuery(parser, metadata, size, depth, backtracks); const ASTCreateQuery & create = query->as(); if (!create.storage || !create.storage->engine) @@ -1479,7 +1491,13 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node { ParserCreateQuery parser; String description = "in ZooKeeper " + zookeeper_path + "/metadata/" + node_name; - auto ast = parseQuery(parser, query, description, 0, getContext()->getSettingsRef().max_parser_depth, getContext()->getSettingsRef().max_parser_backtracks); + auto ast = parseQuery( + parser, + query, + description, + 0, + getContext()->getSettingsRef()[Setting::max_parser_depth], + getContext()->getSettingsRef()[Setting::max_parser_backtracks]); auto & create = ast->as(); if (create.uuid == UUIDHelpers::Nil || create.getTable() != TABLE_WITH_UUID_NAME_PLACEHOLDER || create.database) @@ -1818,7 +1836,8 @@ DatabaseReplicated::getTablesForBackup(const FilterByNameFunction & filter, cons for (const auto & [table_name, metadata] : snapshot) { ParserCreateQuery parser; - auto create_table_query = parseQuery(parser, metadata, 0, getContext()->getSettingsRef().max_parser_depth, getContext()->getSettingsRef().max_parser_backtracks); + auto create_table_query = parseQuery( + parser, metadata, 0, getContext()->getSettingsRef()[Setting::max_parser_depth], getContext()->getSettingsRef()[Setting::max_parser_backtracks]); auto & create = create_table_query->as(); create.attach = false; diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 7993a41df53..f5a9ccb187b 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -11,6 +11,10 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 database_replicated_initial_query_timeout_sec; +} namespace ErrorCodes { @@ -312,7 +316,7 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr task->is_initial_query = true; LOG_DEBUG(log, "Waiting for worker thread to process all entries before {}", entry_name); - UInt64 timeout = query_context->getSettingsRef().database_replicated_initial_query_timeout_sec; + UInt64 timeout = query_context->getSettingsRef()[Setting::database_replicated_initial_query_timeout_sec]; { std::unique_lock lock{mutex}; bool processed = wait_current_task_change.wait_for(lock, std::chrono::seconds(timeout), [&]() diff --git a/src/Databases/DatabaseS3.cpp b/src/Databases/DatabaseS3.cpp index d80cc6d0953..10cd9c084b2 100644 --- a/src/Databases/DatabaseS3.cpp +++ b/src/Databases/DatabaseS3.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,11 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} static const std::unordered_set optional_configuration_keys = { "url", @@ -192,7 +198,8 @@ ASTPtr DatabaseS3::getCreateDatabaseQuery() const creation_args += fmt::format(", '{}', '{}'", config.access_key_id.value(), config.secret_access_key.value()); const String query = fmt::format("CREATE DATABASE {} ENGINE = S3({})", backQuoteIfNeed(getDatabaseName()), creation_args); - ASTPtr ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks); + ASTPtr ast + = parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); if (const auto database_comment = getDatabaseComment(); !database_comment.empty()) { diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp index 2b728039632..067d1d4ed0a 100644 --- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp @@ -2,9 +2,10 @@ #if USE_MYSQL +# include +# include # include # include -# include # include # include @@ -25,6 +26,10 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 glob_expansion_max_elements; +} namespace ErrorCodes { @@ -238,7 +243,7 @@ void registerDatabaseMaterializedMySQL(DatabaseFactory & factory) if (engine_name == "MySQL") { - size_t max_addresses = args.context->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = args.context->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); } else diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 3b72f2aeae5..5d4441b3266 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -39,6 +39,12 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 glob_expansion_max_elements; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -181,8 +187,8 @@ ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const String & table_name, Context storage, table_storage_define, true, - static_cast(settings.max_parser_depth), - static_cast(settings.max_parser_backtracks), + static_cast(settings[Setting::max_parser_depth]), + static_cast(settings[Setting::max_parser_backtracks]), throw_on_error); return create_table_query; } @@ -546,7 +552,7 @@ void registerDatabaseMySQL(DatabaseFactory & factory) if (engine_name == "MySQL") { - size_t max_addresses = args.context->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = args.context->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); } else diff --git a/src/Databases/MySQL/FetchTablesColumnsList.cpp b/src/Databases/MySQL/FetchTablesColumnsList.cpp index 0828438cad2..997eb22e00f 100644 --- a/src/Databases/MySQL/FetchTablesColumnsList.cpp +++ b/src/Databases/MySQL/FetchTablesColumnsList.cpp @@ -40,6 +40,10 @@ String toQueryStringWithQuote(const std::vector & quote_list) namespace DB { +namespace Setting +{ + extern const SettingsBool external_table_functions_use_nulls; +} std::map fetchTablesColumnsList( mysqlxx::PoolWithFailover & pool, @@ -108,16 +112,15 @@ std::map fetchTablesColumnsList( { String table_name = table_name_col[i].safeGet(); ColumnDescription column_description( - column_name_col[i].safeGet(), - convertMySQLDataType( - type_support, - column_type_col[i].safeGet(), - settings.external_table_functions_use_nulls && is_nullable_col[i].safeGet(), - is_unsigned_col[i].safeGet(), - char_max_length_col[i].safeGet(), - precision_col[i].safeGet(), - scale_col[i].safeGet()) - ); + column_name_col[i].safeGet(), + convertMySQLDataType( + type_support, + column_type_col[i].safeGet(), + settings[Setting::external_table_functions_use_nulls] && is_nullable_col[i].safeGet(), + is_unsigned_col[i].safeGet(), + char_max_length_col[i].safeGet(), + precision_col[i].safeGet(), + scale_col[i].safeGet())); column_description.comment = column_comment_col[i].safeGet(); tables_and_columns[table_name].add(column_description); diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 1364e9ae2b2..b1679fcf94f 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -37,6 +37,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_global_with_statement; + extern const SettingsBool insert_allow_materialized_columns; +} namespace ErrorCodes { @@ -90,11 +95,11 @@ static constexpr auto MYSQL_BACKGROUND_THREAD_NAME = "MySQLDBSync"; static ContextMutablePtr createQueryContext(ContextPtr context) { Settings new_query_settings = context->getSettingsCopy(); - new_query_settings.insert_allow_materialized_columns = true; + new_query_settings[Setting::insert_allow_materialized_columns] = true; /// To avoid call AST::format /// TODO: We need to implement the format function for MySQLAST - new_query_settings.enable_global_with_statement = false; + new_query_settings[Setting::enable_global_with_statement] = false; auto query_context = Context::createCopy(context); query_context->setSettings(new_query_settings); diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.h b/src/Databases/MySQL/MaterializedMySQLSyncThread.h index 03e558bfd68..f016967fad5 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.h +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.h @@ -6,21 +6,21 @@ # include # include -# include # include # include # include # include -# include -# include # include +# include # include +# include # include # include namespace DB { +struct MaterializeMetadata; /** MySQL table structure and data synchronization thread * diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index ed62398e594..8bab8a82017 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -32,6 +32,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 postgresql_connection_attempt_timeout; +} namespace ErrorCodes { @@ -535,7 +539,7 @@ void registerDatabaseMaterializedPostgreSQL(DatabaseFactory & factory) configuration.port, configuration.username, configuration.password, - args.context->getSettingsRef().postgresql_connection_attempt_timeout); + args.context->getSettingsRef()[Setting::postgresql_connection_attempt_timeout]); auto postgresql_replica_settings = std::make_unique(); if (engine_define->settings) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 0eafd1c3b5b..8d5a6998d80 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -30,6 +30,15 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 glob_expansion_max_elements; + extern const SettingsUInt64 postgresql_connection_pool_size; + extern const SettingsUInt64 postgresql_connection_pool_wait_timeout; + extern const SettingsUInt64 postgresql_connection_pool_retries; + extern const SettingsBool postgresql_connection_pool_auto_close_connection; + extern const SettingsUInt64 postgresql_connection_attempt_timeout; +} namespace ErrorCodes { @@ -513,7 +522,7 @@ void registerDatabasePostgreSQL(DatabaseFactory & factory) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.context); const auto & host_port = safeGetLiteralValue(engine_args[0], engine_name); - size_t max_addresses = args.context->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = args.context->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 5432); configuration.database = safeGetLiteralValue(engine_args[1], engine_name); @@ -543,11 +552,11 @@ void registerDatabasePostgreSQL(DatabaseFactory & factory) const auto & settings = args.context->getSettingsRef(); auto pool = std::make_shared( configuration, - settings.postgresql_connection_pool_size, - settings.postgresql_connection_pool_wait_timeout, - settings.postgresql_connection_pool_retries, - settings.postgresql_connection_pool_auto_close_connection, - settings.postgresql_connection_attempt_timeout); + settings[Setting::postgresql_connection_pool_size], + settings[Setting::postgresql_connection_pool_wait_timeout], + settings[Setting::postgresql_connection_pool_retries], + settings[Setting::postgresql_connection_pool_auto_close_connection], + settings[Setting::postgresql_connection_attempt_timeout]); return std::make_shared( args.context, diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index 5af9eb1920e..fb01baae7c2 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -17,6 +17,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -198,8 +203,13 @@ ASTPtr DatabaseSQLite::getCreateTableQueryImpl(const String & table_name, Contex const Settings & settings = getContext()->getSettingsRef(); - auto create_table_query = DB::getCreateQueryFromStorage(storage, table_storage_define, true, - static_cast(settings.max_parser_depth), static_cast(settings.max_parser_backtracks), throw_on_error); + auto create_table_query = DB::getCreateQueryFromStorage( + storage, + table_storage_define, + true, + static_cast(settings[Setting::max_parser_depth]), + static_cast(settings[Setting::max_parser_backtracks]), + throw_on_error); return create_table_query; } diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index b3bf288ef5a..085eedf053d 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -1,9 +1,10 @@ #include "CassandraDictionarySource.h" +#include +#include +#include +#include #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" -#include -#include -#include namespace DB { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index b36d53a6159..c902fe81fda 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -1,6 +1,7 @@ #include "ClickHouseDictionarySource.h" #include #include +#include #include #include #include diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index f3b32402c20..79e5857ec69 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -20,6 +20,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; +} namespace ErrorCodes { @@ -414,7 +418,7 @@ void DirectDictionary::applySettings(const Settings & setti if (const auto * clickhouse_source = dynamic_cast(source_ptr.get())) { /// Only applicable for CLICKHOUSE dictionary source. - use_async_executor = settings.dictionary_use_async_executor && clickhouse_source->isLocal(); + use_async_executor = settings[Setting::dictionary_use_async_executor] && clickhouse_source->isLocal(); } } diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index 217a67453e4..403ce540e76 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -23,6 +23,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -205,7 +209,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) size_t max_command_execution_time = config.getUInt64(settings_config_prefix + ".max_command_execution_time", 10); - size_t max_execution_time_seconds = static_cast(context->getSettingsRef().max_execution_time.totalSeconds()); + size_t max_execution_time_seconds = static_cast(context->getSettingsRef()[Setting::max_execution_time].totalSeconds()); if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index bf19f912723..03eca365e2e 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -1,4 +1,5 @@ #include "HTTPDictionarySource.h" +#include #include #include #include diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 8768be8e5ec..83350af2fff 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -17,6 +17,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -1179,10 +1184,10 @@ void registerDictionaryArrayHashed(DictionaryFactory & factory) const auto & settings = context->getSettingsRef(); const auto * clickhouse_source = dynamic_cast(source_ptr.get()); - configuration.use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings.dictionary_use_async_executor; + configuration.use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings[Setting::dictionary_use_async_executor]; - if (settings.max_execution_time.totalSeconds() > 0) - configuration.load_timeout = std::chrono::seconds(settings.max_execution_time.totalSeconds()); + if (settings[Setting::max_execution_time].totalSeconds() > 0) + configuration.load_timeout = std::chrono::seconds(settings[Setting::max_execution_time].totalSeconds()); if (dictionary_key_type == DictionaryKeyType::Simple) { diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index 4f9e991752f..e42ffa09e34 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -29,6 +29,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -1187,7 +1192,8 @@ void registerDictionaryTrie(DictionaryFactory & factory) auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); const auto * clickhouse_source = dynamic_cast(source_ptr.get()); - bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef().dictionary_use_async_executor; + bool use_async_executor + = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef()[Setting::dictionary_use_async_executor]; IPAddressDictionary::Configuration configuration{ .dict_lifetime = dict_lifetime, diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 7bacfdab3d2..0b552f38fd4 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -1,8 +1,9 @@ #include "MongoDBDictionarySource.h" +#include +#include +#include #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" -#include -#include namespace DB { diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 9a84512fde6..a42ccb4f22c 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -10,6 +10,7 @@ #include "DictionaryStructure.h" #include "registerDictionaries.h" #include +#include #include #include #include @@ -28,6 +29,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 external_storage_connect_timeout_sec; + extern const SettingsUInt64 external_storage_rw_timeout_sec; + extern const SettingsUInt64 glob_expansion_max_elements; +} [[maybe_unused]] static const size_t default_num_tries_on_connection_loss = 3; @@ -89,7 +96,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) } else { - size_t max_addresses = global_context->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = global_context->getSettingsRef()[Setting::glob_expansion_max_elements]; addresses = parseRemoteDescriptionForExternalDatabase(addresses_expr, max_addresses, 3306); } @@ -109,9 +116,9 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) const auto & settings = global_context->getSettingsRef(); if (!mysql_settings.isChanged("connect_timeout")) - mysql_settings.connect_timeout = settings.external_storage_connect_timeout_sec; + mysql_settings.connect_timeout = settings[Setting::external_storage_connect_timeout_sec]; if (!mysql_settings.isChanged("read_write_timeout")) - mysql_settings.read_write_timeout = settings.external_storage_rw_timeout_sec; + mysql_settings.read_write_timeout = settings[Setting::external_storage_rw_timeout_sec]; for (const auto & setting : mysql_settings.all()) { diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index 84abeed2731..a01ef1b2717 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -14,6 +14,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; +} namespace ErrorCodes { @@ -223,7 +227,7 @@ DictionaryPtr createLayout(const std::string & , ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); const auto * clickhouse_source = dynamic_cast(source_ptr.get()); - bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef().dictionary_use_async_executor; + bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef()[Setting::dictionary_use_async_executor]; IPolygonDictionary::Configuration configuration { diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index b1bab17e2e9..b5b83370dc6 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -9,6 +9,7 @@ #if USE_LIBPQXX #include +#include #include #include #include "readInvalidateQuery.h" @@ -20,6 +21,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 postgresql_connection_attempt_timeout; + extern const SettingsBool postgresql_connection_pool_auto_close_connection; + extern const SettingsUInt64 postgresql_connection_pool_retries; + extern const SettingsUInt64 postgresql_connection_pool_size; + extern const SettingsUInt64 postgresql_connection_pool_wait_timeout; +} namespace ErrorCodes { @@ -308,11 +317,11 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) auto pool = std::make_shared( replicas_by_priority, - settings.postgresql_connection_pool_size, - settings.postgresql_connection_pool_wait_timeout, - settings.postgresql_connection_pool_retries, - settings.postgresql_connection_pool_auto_close_connection, - settings.postgresql_connection_attempt_timeout); + settings[Setting::postgresql_connection_pool_size], + settings[Setting::postgresql_connection_pool_wait_timeout], + settings[Setting::postgresql_connection_pool_retries], + settings[Setting::postgresql_connection_pool_auto_close_connection], + settings[Setting::postgresql_connection_attempt_timeout]); return std::make_unique(dict_struct, dictionary_configuration.value(), pool, sample_block); diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 9db639a0ca4..17ed515ca9a 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -2,9 +2,10 @@ #include "DictionarySourceFactory.h" #include "DictionaryStructure.h" -#include #include #include +#include +#include #include diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 2522de60183..f5ff50aba8a 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -41,6 +41,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; + extern const SettingsBool regexp_dict_allow_hyperscan; + extern const SettingsBool regexp_dict_flag_case_insensitive; + extern const SettingsBool regexp_dict_flag_dotall; +} namespace ErrorCodes { @@ -990,7 +997,8 @@ void registerDictionaryRegExpTree(DictionaryFactory & factory) auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); const auto * clickhouse_source = typeid_cast(source_ptr.get()); - bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef().dictionary_use_async_executor; + bool use_async_executor + = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef()[Setting::dictionary_use_async_executor]; RegExpTreeDictionary::Configuration configuration{ .require_nonempty = config.getBool(config_prefix + ".require_nonempty", false), @@ -1003,9 +1011,9 @@ void registerDictionaryRegExpTree(DictionaryFactory & factory) dict_struct, std::move(source_ptr), configuration, - context->getSettingsRef().regexp_dict_allow_hyperscan, - context->getSettingsRef().regexp_dict_flag_case_insensitive, - context->getSettingsRef().regexp_dict_flag_dotall); + context->getSettingsRef()[Setting::regexp_dict_allow_hyperscan], + context->getSettingsRef()[Setting::regexp_dict_flag_case_insensitive], + context->getSettingsRef()[Setting::regexp_dict_flag_dotall]); }; factory.registerLayout("regexp_tree", create_layout, true); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 590d000f16e..63e2c8766d5 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -24,6 +24,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds http_receive_timeout; + extern const SettingsBool odbc_bridge_use_connection_pooling; +} + namespace ErrorCodes { extern const int SUPPORT_IS_DISABLED; @@ -241,10 +247,10 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) #if USE_ODBC BridgeHelperPtr bridge = std::make_shared>( global_context, - global_context->getSettingsRef().http_receive_timeout, + global_context->getSettingsRef()[Setting::http_receive_timeout], config.getString(config_prefix + ".odbc.connection_string"), config.getBool(config_prefix + ".settings.odbc_bridge_use_connection_pooling", - global_context->getSettingsRef().odbc_bridge_use_connection_pooling)); + global_context->getSettingsRef()[Setting::odbc_bridge_use_connection_pooling])); std::string settings_config_prefix = config_prefix + ".odbc"; diff --git a/src/Dictionaries/registerCacheDictionaries.cpp b/src/Dictionaries/registerCacheDictionaries.cpp index f2d027575f5..8cfdc72e7c2 100644 --- a/src/Dictionaries/registerCacheDictionaries.cpp +++ b/src/Dictionaries/registerCacheDictionaries.cpp @@ -11,6 +11,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; +} namespace ErrorCodes { @@ -230,7 +234,7 @@ DictionaryPtr createCacheDictionaryLayout( const auto & settings = context->getSettingsRef(); const auto * clickhouse_source = dynamic_cast(source_ptr.get()); - bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings.dictionary_use_async_executor; + bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings[Setting::dictionary_use_async_executor]; CacheDictionaryConfiguration configuration{ allow_read_expired_keys, dict_lifetime, diff --git a/src/Dictionaries/registerHashedDictionary.cpp b/src/Dictionaries/registerHashedDictionary.cpp index 068b71170cc..c571989a641 100644 --- a/src/Dictionaries/registerHashedDictionary.cpp +++ b/src/Dictionaries/registerHashedDictionary.cpp @@ -7,6 +7,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -68,7 +73,7 @@ void registerDictionaryHashed(DictionaryFactory & factory) const auto & settings = context->getSettingsRef(); const auto * clickhouse_source = dynamic_cast(source_ptr.get()); - bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings.dictionary_use_async_executor; + bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && settings[Setting::dictionary_use_async_executor]; HashedDictionaryConfiguration configuration{ static_cast(shards), @@ -77,7 +82,7 @@ void registerDictionaryHashed(DictionaryFactory & factory) require_nonempty, dict_lifetime, use_async_executor, - std::chrono::seconds(settings.max_execution_time.totalSeconds()), + std::chrono::seconds(settings[Setting::max_execution_time].totalSeconds()), }; if (source_ptr->hasUpdateField() && shards > 1) diff --git a/src/Dictionaries/registerRangeHashedDictionary.cpp b/src/Dictionaries/registerRangeHashedDictionary.cpp index bc142a6bddc..13944daef41 100644 --- a/src/Dictionaries/registerRangeHashedDictionary.cpp +++ b/src/Dictionaries/registerRangeHashedDictionary.cpp @@ -8,6 +8,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_use_async_executor; +} namespace ErrorCodes { @@ -59,7 +63,7 @@ static DictionaryPtr createRangeHashedDictionary(const std::string & full_name, auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); const auto * clickhouse_source = dynamic_cast(source_ptr.get()); - bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef().dictionary_use_async_executor; + bool use_async_executor = clickhouse_source && clickhouse_source->isLocal() && context->getSettingsRef()[Setting::dictionary_use_async_executor]; RangeHashedDictionaryConfiguration configuration { diff --git a/src/Disks/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp index 837452aa9f2..1586f0fe679 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds http_connection_timeout; + extern const SettingsSeconds http_receive_timeout; +} namespace ErrorCodes { @@ -53,8 +58,8 @@ std::unique_ptr ReadBufferFromWebServer::initialize() const auto & server_settings = context->getServerSettings(); auto connection_timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, server_settings.keep_alive_timeout); - connection_timeouts.withConnectionTimeout(std::max(settings.http_connection_timeout, Poco::Timespan(20, 0))); - connection_timeouts.withReceiveTimeout(std::max(settings.http_receive_timeout, Poco::Timespan(20, 0))); + connection_timeouts.withConnectionTimeout(std::max(settings[Setting::http_connection_timeout], Poco::Timespan(20, 0))); + connection_timeouts.withReceiveTimeout(std::max(settings[Setting::http_receive_timeout], Poco::Timespan(20, 0))); auto res = BuilderRWBufferFromHTTP(uri) .withConnectionGroup(HTTPConnectionGroupType::DISK) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 1a0b6157a86..931deed30ce 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -3,6 +3,7 @@ #if USE_AZURE_BLOB_STORAGE #include +#include #include #include #include @@ -21,6 +22,24 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 azure_max_single_part_upload_size; + extern const SettingsUInt64 azure_max_single_read_retries; + extern const SettingsUInt64 azure_list_object_keys_size; + extern const SettingsUInt64 azure_min_upload_part_size; + extern const SettingsUInt64 azure_max_upload_part_size; + extern const SettingsUInt64 azure_max_single_part_copy_size; + extern const SettingsUInt64 azure_max_blocks_in_multipart_upload; + extern const SettingsUInt64 azure_max_unexpected_write_error_retries; + extern const SettingsUInt64 azure_max_inflight_parts_for_one_file; + extern const SettingsUInt64 azure_strict_upload_part_size; + extern const SettingsUInt64 azure_upload_part_size_multiply_factor; + extern const SettingsUInt64 azure_upload_part_size_multiply_parts_count_threshold; + extern const SettingsUInt64 azure_sdk_max_retries; + extern const SettingsUInt64 azure_sdk_retry_initial_backoff_ms; + extern const SettingsUInt64 azure_sdk_retry_max_backoff_ms; +} namespace ErrorCodes { @@ -317,22 +336,22 @@ std::unique_ptr getRequestSettings(const Settings & query_setti { auto settings = std::make_unique(); - settings->max_single_part_upload_size = query_settings.azure_max_single_part_upload_size; - settings->max_single_read_retries = query_settings.azure_max_single_read_retries; - settings->max_single_download_retries = query_settings.azure_max_single_read_retries; - settings->list_object_keys_size = query_settings.azure_list_object_keys_size; - settings->min_upload_part_size = query_settings.azure_min_upload_part_size; - settings->max_upload_part_size = query_settings.azure_max_upload_part_size; - settings->max_single_part_copy_size = query_settings.azure_max_single_part_copy_size; - settings->max_blocks_in_multipart_upload = query_settings.azure_max_blocks_in_multipart_upload; - settings->max_unexpected_write_error_retries = query_settings.azure_max_unexpected_write_error_retries; - settings->max_inflight_parts_for_one_file = query_settings.azure_max_inflight_parts_for_one_file; - settings->strict_upload_part_size = query_settings.azure_strict_upload_part_size; - settings->upload_part_size_multiply_factor = query_settings.azure_upload_part_size_multiply_factor; - settings->upload_part_size_multiply_parts_count_threshold = query_settings.azure_upload_part_size_multiply_parts_count_threshold; - settings->sdk_max_retries = query_settings.azure_sdk_max_retries; - settings->sdk_retry_initial_backoff_ms = query_settings.azure_sdk_retry_initial_backoff_ms; - settings->sdk_retry_max_backoff_ms = query_settings.azure_sdk_retry_max_backoff_ms; + settings->max_single_part_upload_size = query_settings[Setting::azure_max_single_part_upload_size]; + settings->max_single_read_retries = query_settings[Setting::azure_max_single_read_retries]; + settings->max_single_download_retries = query_settings[Setting::azure_max_single_read_retries]; + settings->list_object_keys_size = query_settings[Setting::azure_list_object_keys_size]; + settings->min_upload_part_size = query_settings[Setting::azure_min_upload_part_size]; + settings->max_upload_part_size = query_settings[Setting::azure_max_upload_part_size]; + settings->max_single_part_copy_size = query_settings[Setting::azure_max_single_part_copy_size]; + settings->max_blocks_in_multipart_upload = query_settings[Setting::azure_max_blocks_in_multipart_upload]; + settings->max_unexpected_write_error_retries = query_settings[Setting::azure_max_unexpected_write_error_retries]; + settings->max_inflight_parts_for_one_file = query_settings[Setting::azure_max_inflight_parts_for_one_file]; + settings->strict_upload_part_size = query_settings[Setting::azure_strict_upload_part_size]; + settings->upload_part_size_multiply_factor = query_settings[Setting::azure_upload_part_size_multiply_factor]; + settings->upload_part_size_multiply_parts_count_threshold = query_settings[Setting::azure_upload_part_size_multiply_parts_count_threshold]; + settings->sdk_max_retries = query_settings[Setting::azure_sdk_max_retries]; + settings->sdk_retry_initial_backoff_ms = query_settings[Setting::azure_sdk_retry_initial_backoff_ms]; + settings->sdk_retry_max_backoff_ms = query_settings[Setting::azure_sdk_retry_max_backoff_ms]; return settings; } @@ -352,23 +371,23 @@ std::unique_ptr getRequestSettings(const Poco::Util::AbstractCo settings->min_bytes_for_seek = config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024); settings->use_native_copy = config.getBool(config_prefix + ".use_native_copy", false); - settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", settings_ref.azure_max_single_part_upload_size); - settings->max_single_read_retries = config.getUInt64(config_prefix + ".max_single_read_retries", settings_ref.azure_max_single_read_retries); - settings->max_single_download_retries = config.getUInt64(config_prefix + ".max_single_download_retries", settings_ref.azure_max_single_read_retries); - settings->list_object_keys_size = config.getUInt64(config_prefix + ".list_object_keys_size", settings_ref.azure_list_object_keys_size); - settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", settings_ref.azure_min_upload_part_size); - settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", settings_ref.azure_max_upload_part_size); - settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", settings_ref.azure_max_single_part_copy_size); - settings->max_blocks_in_multipart_upload = config.getUInt64(config_prefix + ".max_blocks_in_multipart_upload", settings_ref.azure_max_blocks_in_multipart_upload); - settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", settings_ref.azure_max_unexpected_write_error_retries); - settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", settings_ref.azure_max_inflight_parts_for_one_file); - settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", settings_ref.azure_strict_upload_part_size); - settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", settings_ref.azure_upload_part_size_multiply_factor); - settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", settings_ref.azure_upload_part_size_multiply_parts_count_threshold); + settings->max_single_part_upload_size = config.getUInt64(config_prefix + ".max_single_part_upload_size", settings_ref[Setting::azure_max_single_part_upload_size]); + settings->max_single_read_retries = config.getUInt64(config_prefix + ".max_single_read_retries", settings_ref[Setting::azure_max_single_read_retries]); + settings->max_single_download_retries = config.getUInt64(config_prefix + ".max_single_download_retries", settings_ref[Setting::azure_max_single_read_retries]); + settings->list_object_keys_size = config.getUInt64(config_prefix + ".list_object_keys_size", settings_ref[Setting::azure_list_object_keys_size]); + settings->min_upload_part_size = config.getUInt64(config_prefix + ".min_upload_part_size", settings_ref[Setting::azure_min_upload_part_size]); + settings->max_upload_part_size = config.getUInt64(config_prefix + ".max_upload_part_size", settings_ref[Setting::azure_max_upload_part_size]); + settings->max_single_part_copy_size = config.getUInt64(config_prefix + ".max_single_part_copy_size", settings_ref[Setting::azure_max_single_part_copy_size]); + settings->max_blocks_in_multipart_upload = config.getUInt64(config_prefix + ".max_blocks_in_multipart_upload", settings_ref[Setting::azure_max_blocks_in_multipart_upload]); + settings->max_unexpected_write_error_retries = config.getUInt64(config_prefix + ".max_unexpected_write_error_retries", settings_ref[Setting::azure_max_unexpected_write_error_retries]); + settings->max_inflight_parts_for_one_file = config.getUInt64(config_prefix + ".max_inflight_parts_for_one_file", settings_ref[Setting::azure_max_inflight_parts_for_one_file]); + settings->strict_upload_part_size = config.getUInt64(config_prefix + ".strict_upload_part_size", settings_ref[Setting::azure_strict_upload_part_size]); + settings->upload_part_size_multiply_factor = config.getUInt64(config_prefix + ".upload_part_size_multiply_factor", settings_ref[Setting::azure_upload_part_size_multiply_factor]); + settings->upload_part_size_multiply_parts_count_threshold = config.getUInt64(config_prefix + ".upload_part_size_multiply_parts_count_threshold", settings_ref[Setting::azure_upload_part_size_multiply_parts_count_threshold]); - settings->sdk_max_retries = config.getUInt64(config_prefix + ".max_tries", settings_ref.azure_sdk_max_retries); - settings->sdk_retry_initial_backoff_ms = config.getUInt64(config_prefix + ".retry_initial_backoff_ms", settings_ref.azure_sdk_retry_initial_backoff_ms); - settings->sdk_retry_max_backoff_ms = config.getUInt64(config_prefix + ".retry_max_backoff_ms", settings_ref.azure_sdk_retry_max_backoff_ms); + settings->sdk_max_retries = config.getUInt64(config_prefix + ".max_tries", settings_ref[Setting::azure_sdk_max_retries]); + settings->sdk_retry_initial_backoff_ms = config.getUInt64(config_prefix + ".retry_initial_backoff_ms", settings_ref[Setting::azure_sdk_retry_initial_backoff_ms]); + settings->sdk_retry_max_backoff_ms = config.getUInt64(config_prefix + ".retry_max_backoff_ms", settings_ref[Setting::azure_sdk_retry_max_backoff_ms]); if (config.has(config_prefix + ".curl_ip_resolve")) { diff --git a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp index a690ecd2757..d371f862334 100644 --- a/src/Disks/ObjectStorages/MetadataStorageFactory.cpp +++ b/src/Disks/ObjectStorages/MetadataStorageFactory.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp index d42da6dc9c4..49a33dbf85b 100644 --- a/src/Disks/ObjectStorages/ObjectStorageFactory.cpp +++ b/src/Disks/ObjectStorages/ObjectStorageFactory.cpp @@ -32,6 +32,11 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 hdfs_replication; +} + namespace ErrorCodes { extern const int NO_ELEMENTS_IN_CONFIG; @@ -300,8 +305,7 @@ void registerHDFSObjectStorage(ObjectStorageFactory & factory) throw Exception(ErrorCodes::BAD_ARGUMENTS, "HDFS path must ends with '/', but '{}' doesn't.", uri); std::unique_ptr settings = std::make_unique( - config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), - context->getSettingsRef().hdfs_replication); + config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), context->getSettingsRef()[Setting::hdfs_replication]); return createObjectStorage(ObjectStorageType::HDFS, config, config_prefix, uri, std::move(settings), config, /* lazy_initialize */false); }); diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index f26a3a8bd9d..0df498e1a70 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -45,6 +45,10 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool s3_validate_request_settings; +} namespace ErrorCodes { @@ -267,7 +271,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN query_context && !query_context->isBackgroundOperationContext()) { const auto & settings = query_context->getSettingsRef(); - request_settings.updateFromSettings(settings, /* if_changed */true, settings.s3_validate_request_settings); + request_settings.updateFromSettings(settings, /* if_changed */ true, settings[Setting::s3_validate_request_settings]); } ThreadPoolCallbackRunnerUnsafe scheduler; @@ -589,7 +593,8 @@ void S3ObjectStorage::applyNewSettings( ContextPtr context, const ApplyNewSettingsOptions & options) { - auto settings_from_config = getSettings(config, config_prefix, context, uri.uri_str, context->getSettingsRef().s3_validate_request_settings); + auto settings_from_config + = getSettings(config, config_prefix, context, uri.uri_str, context->getSettingsRef()[Setting::s3_validate_request_settings]); auto modified_settings = std::make_unique(*s3_settings.get()); modified_settings->auth_settings.updateIfChanged(settings_from_config->auth_settings); modified_settings->request_settings.updateIfChanged(settings_from_config->request_settings); @@ -617,7 +622,7 @@ std::unique_ptr S3ObjectStorage::cloneObjectStorage( ContextPtr context) { const auto & settings = context->getSettingsRef(); - auto new_s3_settings = getSettings(config, config_prefix, context, uri.uri_str, settings.s3_validate_request_settings); + auto new_s3_settings = getSettings(config, config_prefix, context, uri.uri_str, settings[Setting::s3_validate_request_settings]); auto new_client = getClient(uri, *new_s3_settings, context, for_disk_s3); auto new_uri{uri}; diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index de78ce4594d..f91c2afe416 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -26,6 +26,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_s3_requests_logging; + extern const SettingsUInt64 s3_max_redirects; + extern const SettingsUInt64 s3_retry_attempts; +} + namespace ErrorCodes { extern const int NO_ELEMENTS_IN_CONFIG; @@ -88,9 +95,9 @@ std::unique_ptr getClient( S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( auth_settings.region, context->getRemoteHostFilter(), - static_cast(global_settings.s3_max_redirects), - static_cast(global_settings.s3_retry_attempts), - global_settings.enable_s3_requests_logging, + static_cast(global_settings[Setting::s3_max_redirects]), + static_cast(global_settings[Setting::s3_retry_attempts]), + global_settings[Setting::enable_s3_requests_logging], for_disk_s3, request_settings.get_request_throttler, request_settings.put_request_throttler, diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 2b302dafd54..aa483282ef6 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -16,13 +16,42 @@ #include #include #include +#include #include +#include +#include #include #include namespace DB { +namespace Setting +{ + /// There are way too many format settings to handle extern declarations manually. +#define DECLARE_FORMAT_EXTERN(TYPE, NAME, DEFAULT, DESCRIPTION, FLAGS) \ + extern Settings ## TYPE NAME; +FORMAT_FACTORY_SETTINGS(DECLARE_FORMAT_EXTERN, SKIP_ALIAS) +#undef DECLARE_FORMAT_EXTERN + + extern const SettingsBool allow_experimental_object_type; + extern const SettingsBool allow_experimental_json_type; + extern const SettingsBool http_write_exception_in_output_format; + extern const SettingsBool input_format_parallel_parsing; + extern const SettingsBool log_queries; + extern const SettingsUInt64 max_download_buffer_size; + extern const SettingsMaxThreads max_download_threads; + extern const SettingsSeconds max_execution_time; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsMaxThreads max_parsing_threads; + extern const SettingsUInt64 max_memory_usage; + extern const SettingsUInt64 max_memory_usage_for_user; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_chunk_bytes_for_parallel_parsing; + extern const SettingsBool output_format_parallel_formatting; + extern const SettingsOverflowMode timeout_overflow_mode; + extern const SettingsInt64 zstd_window_log_max; +} namespace ErrorCodes { @@ -65,237 +94,237 @@ FormatSettings getFormatSettings(const ContextPtr & context) return getFormatSettings(context, settings); } -template FormatSettings getFormatSettings(const ContextPtr & context, const Settings & settings) { FormatSettings format_settings; - format_settings.avro.allow_missing_fields = settings.input_format_avro_allow_missing_fields; - format_settings.avro.output_codec = settings.output_format_avro_codec; - format_settings.avro.output_sync_interval = settings.output_format_avro_sync_interval; - format_settings.avro.schema_registry_url = settings.format_avro_schema_registry_url.toString(); - format_settings.avro.string_column_pattern = settings.output_format_avro_string_column_pattern.toString(); - format_settings.avro.output_rows_in_file = settings.output_format_avro_rows_in_file; - format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; - format_settings.csv.allow_single_quotes = settings.format_csv_allow_single_quotes; - format_settings.csv.serialize_tuple_into_separate_columns = settings.output_format_csv_serialize_tuple_into_separate_columns; - format_settings.csv.deserialize_separate_columns_into_tuple = settings.input_format_csv_deserialize_separate_columns_into_tuple; - format_settings.csv.crlf_end_of_line = settings.output_format_csv_crlf_end_of_line; - format_settings.csv.allow_cr_end_of_line = settings.input_format_csv_allow_cr_end_of_line; - format_settings.csv.delimiter = settings.format_csv_delimiter; - format_settings.csv.tuple_delimiter = settings.format_csv_delimiter; - format_settings.csv.empty_as_default = settings.input_format_csv_empty_as_default; - format_settings.csv.enum_as_number = settings.input_format_csv_enum_as_number; - format_settings.csv.null_representation = settings.format_csv_null_representation; - format_settings.csv.arrays_as_nested_csv = settings.input_format_csv_arrays_as_nested_csv; - format_settings.csv.use_best_effort_in_schema_inference = settings.input_format_csv_use_best_effort_in_schema_inference; - format_settings.csv.skip_first_lines = settings.input_format_csv_skip_first_lines; - format_settings.csv.try_detect_header = settings.input_format_csv_detect_header; - format_settings.csv.skip_trailing_empty_lines = settings.input_format_csv_skip_trailing_empty_lines; - format_settings.csv.trim_whitespaces = settings.input_format_csv_trim_whitespaces; - format_settings.csv.allow_whitespace_or_tab_as_delimiter = settings.input_format_csv_allow_whitespace_or_tab_as_delimiter; - format_settings.csv.allow_variable_number_of_columns = settings.input_format_csv_allow_variable_number_of_columns; - format_settings.csv.use_default_on_bad_values = settings.input_format_csv_use_default_on_bad_values; - format_settings.csv.try_infer_numbers_from_strings = settings.input_format_csv_try_infer_numbers_from_strings; - format_settings.csv.try_infer_strings_from_quoted_tuples = settings.input_format_csv_try_infer_strings_from_quoted_tuples; - format_settings.hive_text.fields_delimiter = settings.input_format_hive_text_fields_delimiter; - format_settings.hive_text.collection_items_delimiter = settings.input_format_hive_text_collection_items_delimiter; - format_settings.hive_text.map_keys_delimiter = settings.input_format_hive_text_map_keys_delimiter; - format_settings.hive_text.allow_variable_number_of_columns = settings.input_format_hive_text_allow_variable_number_of_columns; - format_settings.custom.escaping_rule = settings.format_custom_escaping_rule; - format_settings.custom.field_delimiter = settings.format_custom_field_delimiter; - format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter; - format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter; - format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter; - format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter; - format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter; - format_settings.custom.try_detect_header = settings.input_format_custom_detect_header; - format_settings.custom.skip_trailing_empty_lines = settings.input_format_custom_skip_trailing_empty_lines; - format_settings.custom.allow_variable_number_of_columns = settings.input_format_custom_allow_variable_number_of_columns; - format_settings.date_time_input_format = settings.date_time_input_format; - format_settings.date_time_output_format = settings.date_time_output_format; - format_settings.interval.output_format = settings.interval_output_format; - format_settings.input_format_ipv4_default_on_conversion_error = settings.input_format_ipv4_default_on_conversion_error; - format_settings.input_format_ipv6_default_on_conversion_error = settings.input_format_ipv6_default_on_conversion_error; - format_settings.bool_true_representation = settings.bool_true_representation; - format_settings.bool_false_representation = settings.bool_false_representation; - format_settings.enable_streaming = settings.output_format_enable_streaming; - format_settings.import_nested_json = settings.input_format_import_nested_json; - format_settings.input_allow_errors_num = settings.input_format_allow_errors_num; - format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio; - format_settings.json.max_depth = settings.input_format_json_max_depth; - format_settings.json.array_of_rows = settings.output_format_json_array_of_rows; - format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes; - format_settings.json.write_named_tuples_as_objects = settings.output_format_json_named_tuples_as_objects; - format_settings.json.skip_null_value_in_named_tuples = settings.output_format_json_skip_null_value_in_named_tuples; - format_settings.json.read_named_tuples_as_objects = settings.input_format_json_named_tuples_as_objects; - format_settings.json.use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects = settings.input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects; - format_settings.json.defaults_for_missing_elements_in_named_tuple = settings.input_format_json_defaults_for_missing_elements_in_named_tuple; - format_settings.json.ignore_unknown_keys_in_named_tuple = settings.input_format_json_ignore_unknown_keys_in_named_tuple; - format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; - format_settings.json.quote_64bit_floats = settings.output_format_json_quote_64bit_floats; - format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; - format_settings.json.quote_decimals = settings.output_format_json_quote_decimals; - format_settings.json.read_bools_as_numbers = settings.input_format_json_read_bools_as_numbers; - format_settings.json.read_bools_as_strings = settings.input_format_json_read_bools_as_strings; - format_settings.json.read_numbers_as_strings = settings.input_format_json_read_numbers_as_strings; - format_settings.json.read_objects_as_strings = settings.input_format_json_read_objects_as_strings; - format_settings.json.read_arrays_as_strings = settings.input_format_json_read_arrays_as_strings; - format_settings.json.try_infer_numbers_from_strings = settings.input_format_json_try_infer_numbers_from_strings; - format_settings.json.infer_incomplete_types_as_strings = settings.input_format_json_infer_incomplete_types_as_strings; - format_settings.json.validate_types_from_metadata = settings.input_format_json_validate_types_from_metadata; - format_settings.json.validate_utf8 = settings.output_format_json_validate_utf8; - format_settings.json_object_each_row.column_for_object_name = settings.format_json_object_each_row_column_for_object_name; - format_settings.json.allow_deprecated_object_type = context->getSettingsRef().allow_experimental_object_type; - format_settings.json.allow_json_type = context->getSettingsRef().allow_experimental_json_type; - format_settings.json.compact_allow_variable_number_of_columns = settings.input_format_json_compact_allow_variable_number_of_columns; - format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects; - format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence; - format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields; - format_settings.json.empty_as_default = settings.input_format_json_empty_as_default; - format_settings.json.type_json_skip_duplicated_paths = settings.type_json_skip_duplicated_paths; - format_settings.null_as_default = settings.input_format_null_as_default; - format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields; - format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; - format_settings.parquet.row_group_rows = settings.output_format_parquet_row_group_size; - format_settings.parquet.row_group_bytes = settings.output_format_parquet_row_group_size_bytes; - format_settings.parquet.output_version = settings.output_format_parquet_version; - format_settings.parquet.case_insensitive_column_matching = settings.input_format_parquet_case_insensitive_column_matching; - format_settings.parquet.preserve_order = settings.input_format_parquet_preserve_order; - format_settings.parquet.filter_push_down = settings.input_format_parquet_filter_push_down; - format_settings.parquet.use_native_reader = settings.input_format_parquet_use_native_reader; - format_settings.parquet.allow_missing_columns = settings.input_format_parquet_allow_missing_columns; - format_settings.parquet.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference; - format_settings.parquet.output_string_as_string = settings.output_format_parquet_string_as_string; - format_settings.parquet.output_fixed_string_as_fixed_byte_array = settings.output_format_parquet_fixed_string_as_fixed_byte_array; - format_settings.parquet.max_block_size = settings.input_format_parquet_max_block_size; - format_settings.parquet.prefer_block_bytes = settings.input_format_parquet_prefer_block_bytes; - format_settings.parquet.output_compression_method = settings.output_format_parquet_compression_method; - format_settings.parquet.output_compliant_nested_types = settings.output_format_parquet_compliant_nested_types; - format_settings.parquet.use_custom_encoder = settings.output_format_parquet_use_custom_encoder; - format_settings.parquet.parallel_encoding = settings.output_format_parquet_parallel_encoding; - format_settings.parquet.data_page_size = settings.output_format_parquet_data_page_size; - format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; - format_settings.parquet.write_page_index = settings.output_format_parquet_write_page_index; - format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; - format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.color = settings.output_format_pretty_color.valueOr(2); - format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; - format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; - format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; - format_settings.pretty.max_value_width_apply_for_single_value = settings.output_format_pretty_max_value_width_apply_for_single_value; - format_settings.pretty.highlight_digit_groups = settings.output_format_pretty_highlight_digit_groups; - format_settings.pretty.output_format_pretty_row_numbers = settings.output_format_pretty_row_numbers; - format_settings.pretty.output_format_pretty_single_large_number_tip_threshold = settings.output_format_pretty_single_large_number_tip_threshold; - format_settings.pretty.output_format_pretty_display_footer_column_names = settings.output_format_pretty_display_footer_column_names; - format_settings.pretty.output_format_pretty_display_footer_column_names_min_rows = settings.output_format_pretty_display_footer_column_names_min_rows; - format_settings.protobuf.input_flatten_google_wrappers = settings.input_format_protobuf_flatten_google_wrappers; - format_settings.protobuf.output_nullables_with_google_wrappers = settings.output_format_protobuf_nullables_with_google_wrappers; - format_settings.protobuf.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference; - format_settings.protobuf.use_autogenerated_schema = settings.format_protobuf_use_autogenerated_schema; + format_settings.avro.allow_missing_fields = settings[Setting::input_format_avro_allow_missing_fields]; + format_settings.avro.output_codec = settings[Setting::output_format_avro_codec]; + format_settings.avro.output_sync_interval = settings[Setting::output_format_avro_sync_interval]; + format_settings.avro.schema_registry_url = settings[Setting::format_avro_schema_registry_url].toString(); + format_settings.avro.string_column_pattern = settings[Setting::output_format_avro_string_column_pattern].toString(); + format_settings.avro.output_rows_in_file = settings[Setting::output_format_avro_rows_in_file]; + format_settings.csv.allow_double_quotes = settings[Setting::format_csv_allow_double_quotes]; + format_settings.csv.allow_single_quotes = settings[Setting::format_csv_allow_single_quotes]; + format_settings.csv.serialize_tuple_into_separate_columns = settings[Setting::output_format_csv_serialize_tuple_into_separate_columns]; + format_settings.csv.deserialize_separate_columns_into_tuple = settings[Setting::input_format_csv_deserialize_separate_columns_into_tuple]; + format_settings.csv.crlf_end_of_line = settings[Setting::output_format_csv_crlf_end_of_line]; + format_settings.csv.allow_cr_end_of_line = settings[Setting::input_format_csv_allow_cr_end_of_line]; + format_settings.csv.delimiter = settings[Setting::format_csv_delimiter]; + format_settings.csv.tuple_delimiter = settings[Setting::format_csv_delimiter]; + format_settings.csv.empty_as_default = settings[Setting::input_format_csv_empty_as_default]; + format_settings.csv.enum_as_number = settings[Setting::input_format_csv_enum_as_number]; + format_settings.csv.null_representation = settings[Setting::format_csv_null_representation]; + format_settings.csv.arrays_as_nested_csv = settings[Setting::input_format_csv_arrays_as_nested_csv]; + format_settings.csv.use_best_effort_in_schema_inference = settings[Setting::input_format_csv_use_best_effort_in_schema_inference]; + format_settings.csv.skip_first_lines = settings[Setting::input_format_csv_skip_first_lines]; + format_settings.csv.try_detect_header = settings[Setting::input_format_csv_detect_header]; + format_settings.csv.skip_trailing_empty_lines = settings[Setting::input_format_csv_skip_trailing_empty_lines]; + format_settings.csv.trim_whitespaces = settings[Setting::input_format_csv_trim_whitespaces]; + format_settings.csv.allow_whitespace_or_tab_as_delimiter = settings[Setting::input_format_csv_allow_whitespace_or_tab_as_delimiter]; + format_settings.csv.allow_variable_number_of_columns = settings[Setting::input_format_csv_allow_variable_number_of_columns]; + format_settings.csv.use_default_on_bad_values = settings[Setting::input_format_csv_use_default_on_bad_values]; + format_settings.csv.try_infer_numbers_from_strings = settings[Setting::input_format_csv_try_infer_numbers_from_strings]; + format_settings.csv.try_infer_strings_from_quoted_tuples = settings[Setting::input_format_csv_try_infer_strings_from_quoted_tuples]; + format_settings.hive_text.fields_delimiter = settings[Setting::input_format_hive_text_fields_delimiter]; + format_settings.hive_text.collection_items_delimiter = settings[Setting::input_format_hive_text_collection_items_delimiter]; + format_settings.hive_text.map_keys_delimiter = settings[Setting::input_format_hive_text_map_keys_delimiter]; + format_settings.hive_text.allow_variable_number_of_columns = settings[Setting::input_format_hive_text_allow_variable_number_of_columns]; + format_settings.custom.escaping_rule = settings[Setting::format_custom_escaping_rule]; + format_settings.custom.field_delimiter = settings[Setting::format_custom_field_delimiter]; + format_settings.custom.result_after_delimiter = settings[Setting::format_custom_result_after_delimiter]; + format_settings.custom.result_before_delimiter = settings[Setting::format_custom_result_before_delimiter]; + format_settings.custom.row_after_delimiter = settings[Setting::format_custom_row_after_delimiter]; + format_settings.custom.row_before_delimiter = settings[Setting::format_custom_row_before_delimiter]; + format_settings.custom.row_between_delimiter = settings[Setting::format_custom_row_between_delimiter]; + format_settings.custom.try_detect_header = settings[Setting::input_format_custom_detect_header]; + format_settings.custom.skip_trailing_empty_lines = settings[Setting::input_format_custom_skip_trailing_empty_lines]; + format_settings.custom.allow_variable_number_of_columns = settings[Setting::input_format_custom_allow_variable_number_of_columns]; + format_settings.date_time_input_format = settings[Setting::date_time_input_format]; + format_settings.date_time_output_format = settings[Setting::date_time_output_format]; + format_settings.interval.output_format = settings[Setting::interval_output_format]; + format_settings.input_format_ipv4_default_on_conversion_error = settings[Setting::input_format_ipv4_default_on_conversion_error]; + format_settings.input_format_ipv6_default_on_conversion_error = settings[Setting::input_format_ipv6_default_on_conversion_error]; + format_settings.bool_true_representation = settings[Setting::bool_true_representation]; + format_settings.bool_false_representation = settings[Setting::bool_false_representation]; + format_settings.enable_streaming = settings[Setting::output_format_enable_streaming]; + format_settings.import_nested_json = settings[Setting::input_format_import_nested_json]; + format_settings.input_allow_errors_num = settings[Setting::input_format_allow_errors_num]; + format_settings.input_allow_errors_ratio = settings[Setting::input_format_allow_errors_ratio]; + format_settings.json.max_depth = settings[Setting::input_format_json_max_depth]; + format_settings.json.array_of_rows = settings[Setting::output_format_json_array_of_rows]; + format_settings.json.escape_forward_slashes = settings[Setting::output_format_json_escape_forward_slashes]; + format_settings.json.write_named_tuples_as_objects = settings[Setting::output_format_json_named_tuples_as_objects]; + format_settings.json.skip_null_value_in_named_tuples = settings[Setting::output_format_json_skip_null_value_in_named_tuples]; + format_settings.json.read_named_tuples_as_objects = settings[Setting::input_format_json_named_tuples_as_objects]; + format_settings.json.use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects = settings[Setting::input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects]; + format_settings.json.defaults_for_missing_elements_in_named_tuple = settings[Setting::input_format_json_defaults_for_missing_elements_in_named_tuple]; + format_settings.json.ignore_unknown_keys_in_named_tuple = settings[Setting::input_format_json_ignore_unknown_keys_in_named_tuple]; + format_settings.json.quote_64bit_integers = settings[Setting::output_format_json_quote_64bit_integers]; + format_settings.json.quote_64bit_floats = settings[Setting::output_format_json_quote_64bit_floats]; + format_settings.json.quote_denormals = settings[Setting::output_format_json_quote_denormals]; + format_settings.json.quote_decimals = settings[Setting::output_format_json_quote_decimals]; + format_settings.json.read_bools_as_numbers = settings[Setting::input_format_json_read_bools_as_numbers]; + format_settings.json.read_bools_as_strings = settings[Setting::input_format_json_read_bools_as_strings]; + format_settings.json.read_numbers_as_strings = settings[Setting::input_format_json_read_numbers_as_strings]; + format_settings.json.read_objects_as_strings = settings[Setting::input_format_json_read_objects_as_strings]; + format_settings.json.read_arrays_as_strings = settings[Setting::input_format_json_read_arrays_as_strings]; + format_settings.json.try_infer_numbers_from_strings = settings[Setting::input_format_json_try_infer_numbers_from_strings]; + format_settings.json.infer_incomplete_types_as_strings = settings[Setting::input_format_json_infer_incomplete_types_as_strings]; + format_settings.json.validate_types_from_metadata = settings[Setting::input_format_json_validate_types_from_metadata]; + format_settings.json.validate_utf8 = settings[Setting::output_format_json_validate_utf8]; + format_settings.json_object_each_row.column_for_object_name = settings[Setting::format_json_object_each_row_column_for_object_name]; + format_settings.json.allow_deprecated_object_type = context->getSettingsRef()[Setting::allow_experimental_object_type]; + format_settings.json.allow_json_type = context->getSettingsRef()[Setting::allow_experimental_json_type]; + format_settings.json.compact_allow_variable_number_of_columns = settings[Setting::input_format_json_compact_allow_variable_number_of_columns]; + format_settings.json.try_infer_objects_as_tuples = settings[Setting::input_format_json_try_infer_named_tuples_from_objects]; + format_settings.json.throw_on_bad_escape_sequence = settings[Setting::input_format_json_throw_on_bad_escape_sequence]; + format_settings.json.ignore_unnecessary_fields = settings[Setting::input_format_json_ignore_unnecessary_fields]; + format_settings.json.empty_as_default = settings[Setting::input_format_json_empty_as_default]; + format_settings.json.type_json_skip_duplicated_paths = settings[Setting::type_json_skip_duplicated_paths]; + format_settings.null_as_default = settings[Setting::input_format_null_as_default]; + format_settings.force_null_for_omitted_fields = settings[Setting::input_format_force_null_for_omitted_fields]; + format_settings.decimal_trailing_zeros = settings[Setting::output_format_decimal_trailing_zeros]; + format_settings.parquet.row_group_rows = settings[Setting::output_format_parquet_row_group_size]; + format_settings.parquet.row_group_bytes = settings[Setting::output_format_parquet_row_group_size_bytes]; + format_settings.parquet.output_version = settings[Setting::output_format_parquet_version]; + format_settings.parquet.case_insensitive_column_matching = settings[Setting::input_format_parquet_case_insensitive_column_matching]; + format_settings.parquet.preserve_order = settings[Setting::input_format_parquet_preserve_order]; + format_settings.parquet.filter_push_down = settings[Setting::input_format_parquet_filter_push_down]; + format_settings.parquet.use_native_reader = settings[Setting::input_format_parquet_use_native_reader]; + format_settings.parquet.allow_missing_columns = settings[Setting::input_format_parquet_allow_missing_columns]; + format_settings.parquet.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference]; + format_settings.parquet.output_string_as_string = settings[Setting::output_format_parquet_string_as_string]; + format_settings.parquet.output_fixed_string_as_fixed_byte_array = settings[Setting::output_format_parquet_fixed_string_as_fixed_byte_array]; + format_settings.parquet.max_block_size = settings[Setting::input_format_parquet_max_block_size]; + format_settings.parquet.prefer_block_bytes = settings[Setting::input_format_parquet_prefer_block_bytes]; + format_settings.parquet.output_compression_method = settings[Setting::output_format_parquet_compression_method]; + format_settings.parquet.output_compliant_nested_types = settings[Setting::output_format_parquet_compliant_nested_types]; + format_settings.parquet.use_custom_encoder = settings[Setting::output_format_parquet_use_custom_encoder]; + format_settings.parquet.parallel_encoding = settings[Setting::output_format_parquet_parallel_encoding]; + format_settings.parquet.data_page_size = settings[Setting::output_format_parquet_data_page_size]; + format_settings.parquet.write_batch_size = settings[Setting::output_format_parquet_batch_size]; + format_settings.parquet.write_page_index = settings[Setting::output_format_parquet_write_page_index]; + format_settings.parquet.local_read_min_bytes_for_seek = settings[Setting::input_format_parquet_local_file_min_bytes_for_seek]; + format_settings.pretty.charset = settings[Setting::output_format_pretty_grid_charset].toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; + format_settings.pretty.color = settings[Setting::output_format_pretty_color].valueOr(2); + format_settings.pretty.max_column_pad_width = settings[Setting::output_format_pretty_max_column_pad_width]; + format_settings.pretty.max_rows = settings[Setting::output_format_pretty_max_rows]; + format_settings.pretty.max_value_width = settings[Setting::output_format_pretty_max_value_width]; + format_settings.pretty.max_value_width_apply_for_single_value = settings[Setting::output_format_pretty_max_value_width_apply_for_single_value]; + format_settings.pretty.highlight_digit_groups = settings[Setting::output_format_pretty_highlight_digit_groups]; + format_settings.pretty.output_format_pretty_row_numbers = settings[Setting::output_format_pretty_row_numbers]; + format_settings.pretty.output_format_pretty_single_large_number_tip_threshold = settings[Setting::output_format_pretty_single_large_number_tip_threshold]; + format_settings.pretty.output_format_pretty_display_footer_column_names = settings[Setting::output_format_pretty_display_footer_column_names]; + format_settings.pretty.output_format_pretty_display_footer_column_names_min_rows = settings[Setting::output_format_pretty_display_footer_column_names_min_rows]; + format_settings.protobuf.input_flatten_google_wrappers = settings[Setting::input_format_protobuf_flatten_google_wrappers]; + format_settings.protobuf.output_nullables_with_google_wrappers = settings[Setting::output_format_protobuf_nullables_with_google_wrappers]; + format_settings.protobuf.skip_fields_with_unsupported_types_in_schema_inference = settings[Setting::input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference]; + format_settings.protobuf.use_autogenerated_schema = settings[Setting::format_protobuf_use_autogenerated_schema]; format_settings.protobuf.google_protos_path = context->getGoogleProtosPath(); - format_settings.regexp.escaping_rule = settings.format_regexp_escaping_rule; - format_settings.regexp.regexp = settings.format_regexp; - format_settings.regexp.skip_unmatched = settings.format_regexp_skip_unmatched; - format_settings.schema.format_schema = settings.format_schema; + format_settings.regexp.escaping_rule = settings[Setting::format_regexp_escaping_rule]; + format_settings.regexp.regexp = settings[Setting::format_regexp]; + format_settings.regexp.skip_unmatched = settings[Setting::format_regexp_skip_unmatched]; + format_settings.schema.format_schema = settings[Setting::format_schema]; format_settings.schema.format_schema_path = context->getFormatSchemaPath(); format_settings.schema.is_server = context->hasGlobalContext() && (context->getGlobalContext()->getApplicationType() == Context::ApplicationType::SERVER); - format_settings.schema.output_format_schema = settings.output_format_schema; - format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields; - format_settings.template_settings.resultset_format = settings.format_template_resultset; - format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; - format_settings.template_settings.row_format = settings.format_template_row; - format_settings.template_settings.row_format_template = settings.format_template_row_format; - format_settings.template_settings.resultset_format_template = settings.format_template_resultset_format; - format_settings.tsv.crlf_end_of_line = settings.output_format_tsv_crlf_end_of_line; - format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; - format_settings.tsv.enum_as_number = settings.input_format_tsv_enum_as_number; - format_settings.tsv.null_representation = settings.format_tsv_null_representation; - format_settings.tsv.use_best_effort_in_schema_inference = settings.input_format_tsv_use_best_effort_in_schema_inference; - format_settings.tsv.skip_first_lines = settings.input_format_tsv_skip_first_lines; - format_settings.tsv.try_detect_header = settings.input_format_tsv_detect_header; - format_settings.tsv.skip_trailing_empty_lines = settings.input_format_tsv_skip_trailing_empty_lines; - format_settings.tsv.allow_variable_number_of_columns = settings.input_format_tsv_allow_variable_number_of_columns; - format_settings.tsv.crlf_end_of_line_input = settings.input_format_tsv_crlf_end_of_line; - format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals; - format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; - format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; - format_settings.values.escape_quote_with_quote = settings.output_format_values_escape_quote_with_quote; - format_settings.with_names_use_header = settings.input_format_with_names_use_header; - format_settings.with_types_use_header = settings.input_format_with_types_use_header; - format_settings.write_statistics = settings.output_format_write_statistics; - format_settings.arrow.low_cardinality_as_dictionary = settings.output_format_arrow_low_cardinality_as_dictionary; - format_settings.arrow.use_signed_indexes_for_dictionary = settings.output_format_arrow_use_signed_indexes_for_dictionary; - format_settings.arrow.use_64_bit_indexes_for_dictionary = settings.output_format_arrow_use_64_bit_indexes_for_dictionary; - format_settings.arrow.allow_missing_columns = settings.input_format_arrow_allow_missing_columns; - format_settings.arrow.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference; - format_settings.arrow.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference; - format_settings.arrow.case_insensitive_column_matching = settings.input_format_arrow_case_insensitive_column_matching; - format_settings.arrow.output_string_as_string = settings.output_format_arrow_string_as_string; - format_settings.arrow.output_fixed_string_as_fixed_byte_array = settings.output_format_arrow_fixed_string_as_fixed_byte_array; - format_settings.arrow.output_compression_method = settings.output_format_arrow_compression_method; - format_settings.orc.allow_missing_columns = settings.input_format_orc_allow_missing_columns; - format_settings.orc.row_batch_size = settings.input_format_orc_row_batch_size; - format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_orc_skip_columns_with_unsupported_types_in_schema_inference; - format_settings.orc.allow_missing_columns = settings.input_format_orc_allow_missing_columns; - format_settings.orc.row_batch_size = settings.input_format_orc_row_batch_size; - format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_orc_skip_columns_with_unsupported_types_in_schema_inference; - format_settings.orc.case_insensitive_column_matching = settings.input_format_orc_case_insensitive_column_matching; - format_settings.orc.output_string_as_string = settings.output_format_orc_string_as_string; - format_settings.orc.output_compression_method = settings.output_format_orc_compression_method; - format_settings.orc.output_row_index_stride = settings.output_format_orc_row_index_stride; - format_settings.orc.output_dictionary_key_size_threshold = settings.output_format_orc_dictionary_key_size_threshold; - format_settings.orc.use_fast_decoder = settings.input_format_orc_use_fast_decoder; - format_settings.orc.filter_push_down = settings.input_format_orc_filter_push_down; - format_settings.orc.reader_time_zone_name = settings.input_format_orc_reader_time_zone_name; - format_settings.defaults_for_omitted_fields = settings.input_format_defaults_for_omitted_fields; - format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode; - format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference; - format_settings.capn_proto.use_autogenerated_schema = settings.format_capn_proto_use_autogenerated_schema; - format_settings.seekable_read = settings.input_format_allow_seeks; - format_settings.msgpack.number_of_columns = settings.input_format_msgpack_number_of_columns; - format_settings.msgpack.output_uuid_representation = settings.output_format_msgpack_uuid_representation; - format_settings.max_rows_to_read_for_schema_inference = settings.input_format_max_rows_to_read_for_schema_inference; - format_settings.max_bytes_to_read_for_schema_inference = settings.input_format_max_bytes_to_read_for_schema_inference; - format_settings.column_names_for_schema_inference = settings.column_names_for_schema_inference; - format_settings.schema_inference_hints = settings.schema_inference_hints; - format_settings.schema_inference_make_columns_nullable = settings.schema_inference_make_columns_nullable.valueOr(2); - format_settings.mysql_dump.table_name = settings.input_format_mysql_dump_table_name; - format_settings.mysql_dump.map_column_names = settings.input_format_mysql_dump_map_column_names; - format_settings.sql_insert.max_batch_size = settings.output_format_sql_insert_max_batch_size; - format_settings.sql_insert.include_column_names = settings.output_format_sql_insert_include_column_names; - format_settings.sql_insert.table_name = settings.output_format_sql_insert_table_name; - format_settings.sql_insert.use_replace = settings.output_format_sql_insert_use_replace; - format_settings.sql_insert.quote_names = settings.output_format_sql_insert_quote_names; - format_settings.try_infer_integers = settings.input_format_try_infer_integers; - format_settings.try_infer_dates = settings.input_format_try_infer_dates; - format_settings.try_infer_datetimes = settings.input_format_try_infer_datetimes; - format_settings.try_infer_datetimes_only_datetime64 = settings.input_format_try_infer_datetimes_only_datetime64; - format_settings.try_infer_exponent_floats = settings.input_format_try_infer_exponent_floats; - format_settings.markdown.escape_special_characters = settings.output_format_markdown_escape_special_characters; - format_settings.bson.output_string_as_string = settings.output_format_bson_string_as_string; - format_settings.bson.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_bson_skip_fields_with_unsupported_types_in_schema_inference; - format_settings.binary.max_binary_string_size = settings.format_binary_max_string_size; - format_settings.binary.max_binary_array_size = settings.format_binary_max_array_size; - format_settings.binary.encode_types_in_binary_format = settings.output_format_binary_encode_types_in_binary_format; - format_settings.binary.decode_types_in_binary_format = settings.input_format_binary_decode_types_in_binary_format; - format_settings.native.allow_types_conversion = settings.input_format_native_allow_types_conversion; - format_settings.native.encode_types_in_binary_format = settings.output_format_native_encode_types_in_binary_format; - format_settings.native.decode_types_in_binary_format = settings.input_format_native_decode_types_in_binary_format; - format_settings.max_parser_depth = context->getSettingsRef().max_parser_depth; + format_settings.schema.output_format_schema = settings[Setting::output_format_schema]; + format_settings.skip_unknown_fields = settings[Setting::input_format_skip_unknown_fields]; + format_settings.template_settings.resultset_format = settings[Setting::format_template_resultset]; + format_settings.template_settings.row_between_delimiter = settings[Setting::format_template_rows_between_delimiter]; + format_settings.template_settings.row_format = settings[Setting::format_template_row]; + format_settings.template_settings.row_format_template = settings[Setting::format_template_row_format]; + format_settings.template_settings.resultset_format_template = settings[Setting::format_template_resultset_format]; + format_settings.tsv.crlf_end_of_line = settings[Setting::output_format_tsv_crlf_end_of_line]; + format_settings.tsv.empty_as_default = settings[Setting::input_format_tsv_empty_as_default]; + format_settings.tsv.enum_as_number = settings[Setting::input_format_tsv_enum_as_number]; + format_settings.tsv.null_representation = settings[Setting::format_tsv_null_representation]; + format_settings.tsv.use_best_effort_in_schema_inference = settings[Setting::input_format_tsv_use_best_effort_in_schema_inference]; + format_settings.tsv.skip_first_lines = settings[Setting::input_format_tsv_skip_first_lines]; + format_settings.tsv.try_detect_header = settings[Setting::input_format_tsv_detect_header]; + format_settings.tsv.skip_trailing_empty_lines = settings[Setting::input_format_tsv_skip_trailing_empty_lines]; + format_settings.tsv.allow_variable_number_of_columns = settings[Setting::input_format_tsv_allow_variable_number_of_columns]; + format_settings.tsv.crlf_end_of_line_input = settings[Setting::input_format_tsv_crlf_end_of_line]; + format_settings.values.accurate_types_of_literals = settings[Setting::input_format_values_accurate_types_of_literals]; + format_settings.values.deduce_templates_of_expressions = settings[Setting::input_format_values_deduce_templates_of_expressions]; + format_settings.values.interpret_expressions = settings[Setting::input_format_values_interpret_expressions]; + format_settings.values.escape_quote_with_quote = settings[Setting::output_format_values_escape_quote_with_quote]; + format_settings.with_names_use_header = settings[Setting::input_format_with_names_use_header]; + format_settings.with_types_use_header = settings[Setting::input_format_with_types_use_header]; + format_settings.write_statistics = settings[Setting::output_format_write_statistics]; + format_settings.arrow.low_cardinality_as_dictionary = settings[Setting::output_format_arrow_low_cardinality_as_dictionary]; + format_settings.arrow.use_signed_indexes_for_dictionary = settings[Setting::output_format_arrow_use_signed_indexes_for_dictionary]; + format_settings.arrow.use_64_bit_indexes_for_dictionary = settings[Setting::output_format_arrow_use_64_bit_indexes_for_dictionary]; + format_settings.arrow.allow_missing_columns = settings[Setting::input_format_arrow_allow_missing_columns]; + format_settings.arrow.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference]; + format_settings.arrow.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference]; + format_settings.arrow.case_insensitive_column_matching = settings[Setting::input_format_arrow_case_insensitive_column_matching]; + format_settings.arrow.output_string_as_string = settings[Setting::output_format_arrow_string_as_string]; + format_settings.arrow.output_fixed_string_as_fixed_byte_array = settings[Setting::output_format_arrow_fixed_string_as_fixed_byte_array]; + format_settings.arrow.output_compression_method = settings[Setting::output_format_arrow_compression_method]; + format_settings.orc.allow_missing_columns = settings[Setting::input_format_orc_allow_missing_columns]; + format_settings.orc.row_batch_size = settings[Setting::input_format_orc_row_batch_size]; + format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_orc_skip_columns_with_unsupported_types_in_schema_inference]; + format_settings.orc.allow_missing_columns = settings[Setting::input_format_orc_allow_missing_columns]; + format_settings.orc.row_batch_size = settings[Setting::input_format_orc_row_batch_size]; + format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_orc_skip_columns_with_unsupported_types_in_schema_inference]; + format_settings.orc.case_insensitive_column_matching = settings[Setting::input_format_orc_case_insensitive_column_matching]; + format_settings.orc.output_string_as_string = settings[Setting::output_format_orc_string_as_string]; + format_settings.orc.output_compression_method = settings[Setting::output_format_orc_compression_method]; + format_settings.orc.output_row_index_stride = settings[Setting::output_format_orc_row_index_stride]; + format_settings.orc.output_dictionary_key_size_threshold = settings[Setting::output_format_orc_dictionary_key_size_threshold]; + format_settings.orc.use_fast_decoder = settings[Setting::input_format_orc_use_fast_decoder]; + format_settings.orc.filter_push_down = settings[Setting::input_format_orc_filter_push_down]; + format_settings.orc.reader_time_zone_name = settings[Setting::input_format_orc_reader_time_zone_name]; + format_settings.defaults_for_omitted_fields = settings[Setting::input_format_defaults_for_omitted_fields]; + format_settings.capn_proto.enum_comparing_mode = settings[Setting::format_capn_proto_enum_comparising_mode]; + format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings[Setting::input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference]; + format_settings.capn_proto.use_autogenerated_schema = settings[Setting::format_capn_proto_use_autogenerated_schema]; + format_settings.seekable_read = settings[Setting::input_format_allow_seeks]; + format_settings.msgpack.number_of_columns = settings[Setting::input_format_msgpack_number_of_columns]; + format_settings.msgpack.output_uuid_representation = settings[Setting::output_format_msgpack_uuid_representation]; + format_settings.max_rows_to_read_for_schema_inference = settings[Setting::input_format_max_rows_to_read_for_schema_inference]; + format_settings.max_bytes_to_read_for_schema_inference = settings[Setting::input_format_max_bytes_to_read_for_schema_inference]; + format_settings.column_names_for_schema_inference = settings[Setting::column_names_for_schema_inference]; + format_settings.schema_inference_hints = settings[Setting::schema_inference_hints]; + format_settings.schema_inference_make_columns_nullable = settings[Setting::schema_inference_make_columns_nullable].valueOr(2); + format_settings.mysql_dump.table_name = settings[Setting::input_format_mysql_dump_table_name]; + format_settings.mysql_dump.map_column_names = settings[Setting::input_format_mysql_dump_map_column_names]; + format_settings.sql_insert.max_batch_size = settings[Setting::output_format_sql_insert_max_batch_size]; + format_settings.sql_insert.include_column_names = settings[Setting::output_format_sql_insert_include_column_names]; + format_settings.sql_insert.table_name = settings[Setting::output_format_sql_insert_table_name]; + format_settings.sql_insert.use_replace = settings[Setting::output_format_sql_insert_use_replace]; + format_settings.sql_insert.quote_names = settings[Setting::output_format_sql_insert_quote_names]; + format_settings.try_infer_integers = settings[Setting::input_format_try_infer_integers]; + format_settings.try_infer_dates = settings[Setting::input_format_try_infer_dates]; + format_settings.try_infer_datetimes = settings[Setting::input_format_try_infer_datetimes]; + format_settings.try_infer_datetimes_only_datetime64 = settings[Setting::input_format_try_infer_datetimes_only_datetime64]; + format_settings.try_infer_exponent_floats = settings[Setting::input_format_try_infer_exponent_floats]; + format_settings.markdown.escape_special_characters = settings[Setting::output_format_markdown_escape_special_characters]; + format_settings.bson.output_string_as_string = settings[Setting::output_format_bson_string_as_string]; + format_settings.bson.skip_fields_with_unsupported_types_in_schema_inference = settings[Setting::input_format_bson_skip_fields_with_unsupported_types_in_schema_inference]; + format_settings.binary.max_binary_string_size = settings[Setting::format_binary_max_string_size]; + format_settings.binary.max_binary_array_size = settings[Setting::format_binary_max_array_size]; + format_settings.binary.encode_types_in_binary_format = settings[Setting::output_format_binary_encode_types_in_binary_format]; + format_settings.binary.decode_types_in_binary_format = settings[Setting::input_format_binary_decode_types_in_binary_format]; + format_settings.native.allow_types_conversion = settings[Setting::input_format_native_allow_types_conversion]; + format_settings.native.encode_types_in_binary_format = settings[Setting::output_format_native_encode_types_in_binary_format]; + format_settings.native.decode_types_in_binary_format = settings[Setting::input_format_native_decode_types_in_binary_format]; + format_settings.max_parser_depth = context->getSettingsRef()[Setting::max_parser_depth]; format_settings.client_protocol_version = context->getClientProtocolVersion(); - format_settings.date_time_overflow_behavior = settings.date_time_overflow_behavior; - format_settings.try_infer_variant = settings.input_format_try_infer_variants; + format_settings.date_time_overflow_behavior = settings[Setting::date_time_overflow_behavior]; + format_settings.try_infer_variant = settings[Setting::input_format_try_infer_variants]; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context if (format_settings.schema.is_server) { - const Poco::URI & avro_schema_registry_url = settings.format_avro_schema_registry_url; + const Poco::URI & avro_schema_registry_url = settings[Setting::format_avro_schema_registry_url]; if (!avro_schema_registry_url.empty()) context->getRemoteHostFilter().checkURL(avro_schema_registry_url); } - if (context->getClientInfo().interface == ClientInfo::Interface::HTTP && context->getSettingsRef().http_write_exception_in_output_format.value) + if (context->getClientInfo().interface == ClientInfo::Interface::HTTP + && context->getSettingsRef()[Setting::http_write_exception_in_output_format].value) { format_settings.json.valid_output_on_exception = true; format_settings.xml.valid_output_on_exception = true; @@ -304,10 +333,6 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se return format_settings; } -template FormatSettings getFormatSettings(const ContextPtr & context, const FormatFactorySettings & settings); - -template FormatSettings getFormatSettings(const ContextPtr & context, const Settings & settings); - InputFormatPtr FormatFactory::getInput( const String & name, @@ -328,17 +353,17 @@ InputFormatPtr FormatFactory::getInput( auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); const Settings & settings = context->getSettingsRef(); - size_t max_parsing_threads = _max_parsing_threads.value_or(settings.max_parsing_threads); - size_t max_download_threads = _max_download_threads.value_or(settings.max_download_threads); + size_t max_parsing_threads = _max_parsing_threads.value_or(settings[Setting::max_parsing_threads]); + size_t max_download_threads = _max_download_threads.value_or(settings[Setting::max_download_threads]); RowInputFormatParams row_input_format_params; row_input_format_params.max_block_size = max_block_size; row_input_format_params.allow_errors_num = format_settings.input_allow_errors_num; row_input_format_params.allow_errors_ratio = format_settings.input_allow_errors_ratio; - row_input_format_params.max_execution_time = settings.max_execution_time; - row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode; + row_input_format_params.max_execution_time = settings[Setting::max_execution_time]; + row_input_format_params.timeout_overflow_mode = settings[Setting::timeout_overflow_mode]; - if (context->hasQueryContext() && settings.log_queries) + if (context->hasQueryContext() && settings[Setting::log_queries]) context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Format, name); // Add ParallelReadBuffer and decompression if needed. @@ -348,13 +373,14 @@ InputFormatPtr FormatFactory::getInput( // Decide whether to use ParallelParsingInputFormat. - bool parallel_parsing = - max_parsing_threads > 1 && settings.input_format_parallel_parsing && creators.file_segmentation_engine_creator && - !creators.random_access_input_creator && !need_only_count; + bool parallel_parsing = max_parsing_threads > 1 && settings[Setting::input_format_parallel_parsing] + && creators.file_segmentation_engine_creator && !creators.random_access_input_creator && !need_only_count; - if (settings.max_memory_usage && settings.min_chunk_bytes_for_parallel_parsing * max_parsing_threads * 2 > settings.max_memory_usage) + if (settings[Setting::max_memory_usage] + && settings[Setting::min_chunk_bytes_for_parallel_parsing] * max_parsing_threads * 2 > settings[Setting::max_memory_usage]) parallel_parsing = false; - if (settings.max_memory_usage_for_user && settings.min_chunk_bytes_for_parallel_parsing * max_parsing_threads * 2 > settings.max_memory_usage_for_user) + if (settings[Setting::max_memory_usage_for_user] + && settings[Setting::min_chunk_bytes_for_parallel_parsing] * max_parsing_threads * 2 > settings[Setting::max_memory_usage_for_user]) parallel_parsing = false; if (parallel_parsing) @@ -379,21 +405,23 @@ InputFormatPtr FormatFactory::getInput( { return input_getter(input, sample, row_input_format_params, format_settings); }; ParallelParsingInputFormat::Params params{ - buf, sample, parser_creator, creators.file_segmentation_engine_creator, name, format_settings, max_parsing_threads, - settings.min_chunk_bytes_for_parallel_parsing, max_block_size, context->getApplicationType() == Context::ApplicationType::SERVER}; + buf, + sample, + parser_creator, + creators.file_segmentation_engine_creator, + name, + format_settings, + max_parsing_threads, + settings[Setting::min_chunk_bytes_for_parallel_parsing], + max_block_size, + context->getApplicationType() == Context::ApplicationType::SERVER}; format = std::make_shared(params); } else if (creators.random_access_input_creator) { format = creators.random_access_input_creator( - buf, - sample, - format_settings, - context->getReadSettings(), - is_remote_fs, - max_download_threads, - max_parsing_threads); + buf, sample, format_settings, context->getReadSettings(), is_remote_fs, max_download_threads, max_parsing_threads); } else { @@ -402,7 +430,7 @@ InputFormatPtr FormatFactory::getInput( if (owned_buf) format->addBuffer(std::move(owned_buf)); - if (!settings.input_format_record_errors_file_path.toString().empty()) + if (!settings[Setting::input_format_record_errors_file_path].toString().empty()) { if (parallel_parsing) format->setErrorsLogger(std::make_shared(context)); @@ -440,7 +468,7 @@ std::unique_ptr FormatFactory::wrapReadBufferIfNeeded( try { file_size = getFileSizeFromReadBuffer(buf); - parallel_read = file_size >= 2 * settings.max_download_buffer_size; + parallel_read = file_size >= 2 * settings[Setting::max_download_buffer_size]; } catch (const Poco::Exception & e) { @@ -459,18 +487,21 @@ std::unique_ptr FormatFactory::wrapReadBufferIfNeeded( getLogger("FormatFactory"), "Using ParallelReadBuffer with {} workers with chunks of {} bytes", max_download_threads, - settings.max_download_buffer_size); + settings[Setting::max_download_buffer_size]); res = wrapInParallelReadBufferIfSupported( - buf, threadPoolCallbackRunnerUnsafe(getIOThreadPool().get(), "ParallelRead"), - max_download_threads, settings.max_download_buffer_size, file_size); + buf, + threadPoolCallbackRunnerUnsafe(getIOThreadPool().get(), "ParallelRead"), + max_download_threads, + settings[Setting::max_download_buffer_size], + file_size); } if (compression != CompressionMethod::None) { if (!res) res = wrapReadBufferReference(buf); - res = wrapReadBufferWithCompressionMethod(std::move(res), compression, static_cast(settings.zstd_window_log_max)); + res = wrapReadBufferWithCompressionMethod(std::move(res), compression, static_cast(settings[Setting::zstd_window_log_max])); } return res; @@ -507,17 +538,17 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( const Settings & settings = context->getSettingsRef(); - if (settings.output_format_parallel_formatting && getCreators(name).supports_parallel_formatting - && !settings.output_format_json_array_of_rows) + if (settings[Setting::output_format_parallel_formatting] && getCreators(name).supports_parallel_formatting + && !settings[Setting::output_format_json_array_of_rows]) { auto formatter_creator = [output_getter, sample, format_settings] (WriteBuffer & output) -> OutputFormatPtr { return output_getter(output, sample, format_settings); }; - ParallelFormattingOutputFormat::Params builder{buf, sample, formatter_creator, settings.max_threads}; + ParallelFormattingOutputFormat::Params builder{buf, sample, formatter_creator, settings[Setting::max_threads]}; - if (context->hasQueryContext() && settings.log_queries) + if (context->hasQueryContext() && settings[Setting::log_queries]) context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Format, name); auto format = std::make_shared(builder); @@ -540,11 +571,11 @@ OutputFormatPtr FormatFactory::getOutputFormat( if (!output_getter) throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output", name); - if (context->hasQueryContext() && context->getSettingsRef().log_queries) + if (context->hasQueryContext() && context->getSettingsRef()[Setting::log_queries]) context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Format, name); auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); - format_settings.max_threads = context->getSettingsRef().max_threads; + format_settings.max_threads = context->getSettingsRef()[Setting::max_threads]; format_settings.is_writing_to_terminal = format_settings.is_writing_to_terminal = isWritingToTerminal(buf); /** TODO: Materialization is needed, because formats can use the functions `IDataType`, @@ -865,7 +896,7 @@ bool FormatFactory::checkIfOutputFormatPrefersLargeBlocks(const String & name) c bool FormatFactory::checkParallelizeOutputAfterReading(const String & name, const ContextPtr & context) const { auto format_name = boost::to_lower_copy(name); - if (format_name == "parquet" && context->getSettingsRef().input_format_parquet_preserve_order) + if (format_name == "parquet" && context->getSettingsRef()[Setting::input_format_parquet_preserve_order]) return false; return true; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 93f67e83831..ad0dd30a60c 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -22,6 +22,7 @@ namespace DB class Block; struct Settings; +class SettingsChanges; struct FormatFactorySettings; struct ReadSettings; @@ -50,9 +51,7 @@ template struct Memory; FormatSettings getFormatSettings(const ContextPtr & context); - -template -FormatSettings getFormatSettings(const ContextPtr & context, const T & settings); +FormatSettings getFormatSettings(const ContextPtr & context, const Settings & settings); /** Allows to create an IInputFormat or IOutputFormat by the name of the format. * Note: format and compression are independent things. @@ -95,13 +94,13 @@ private: // Incompatible with FileSegmentationEngine. using RandomAccessInputCreator = std::function; + ReadBuffer & buf, + const Block & header, + const FormatSettings & settings, + const ReadSettings & read_settings, + bool is_remote_fs, + size_t max_download_threads, + size_t max_parsing_threads)>; using OutputCreator = std::function #include #include +#include #include + #include diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index dbbd728ed72..6ad9fc0499a 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -7,10 +9,15 @@ #include #include #include -#include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 input_format_max_rows_to_read_for_schema_inference; + extern const SettingsUInt64 input_format_max_bytes_to_read_for_schema_inference; + extern const SettingsSchemaInferenceMode schema_inference_mode; +} namespace ErrorCodes { @@ -99,7 +106,7 @@ std::pair readSchemaFromFormatImpl( try { NamesAndTypesList names_and_types; - SchemaInferenceMode mode = context->getSettingsRef().schema_inference_mode; + SchemaInferenceMode mode = context->getSettingsRef()[Setting::schema_inference_mode]; if (format_name && mode == SchemaInferenceMode::UNION && !FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(*format_name, context, format_settings)) { String additional_message; @@ -131,9 +138,9 @@ try std::vector> schemas_for_union_mode; std::string exception_messages; size_t max_rows_to_read = format_settings ? format_settings->max_rows_to_read_for_schema_inference - : context->getSettingsRef().input_format_max_rows_to_read_for_schema_inference; + : context->getSettingsRef()[Setting::input_format_max_rows_to_read_for_schema_inference]; size_t max_bytes_to_read = format_settings ? format_settings->max_bytes_to_read_for_schema_inference - : context->getSettingsRef().input_format_max_bytes_to_read_for_schema_inference; + : context->getSettingsRef()[Setting::input_format_max_bytes_to_read_for_schema_inference]; size_t iterations = 0; while (true) { @@ -570,7 +577,7 @@ SchemaCache::Keys getKeysForSchemaCache( /// For example, for Protobuf format additional information is the path to the schema /// and message name. String additional_format_info = FormatFactory::instance().getAdditionalInfoForSchemaCache(format, context, format_settings); - String schema_inference_mode(magic_enum::enum_name(context->getSettingsRef().schema_inference_mode.value)); + String schema_inference_mode(magic_enum::enum_name(context->getSettingsRef()[Setting::schema_inference_mode].value)); SchemaCache::Keys cache_keys; cache_keys.reserve(sources.size()); std::transform( diff --git a/src/Formats/ReadSchemaUtils.h b/src/Formats/ReadSchemaUtils.h index 7168e7f0817..4c4e788100a 100644 --- a/src/Formats/ReadSchemaUtils.h +++ b/src/Formats/ReadSchemaUtils.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/src/Formats/registerWithNamesAndTypes.cpp b/src/Formats/registerWithNamesAndTypes.cpp index 674865a3bed..606f445b654 100644 --- a/src/Formats/registerWithNamesAndTypes.cpp +++ b/src/Formats/registerWithNamesAndTypes.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB diff --git a/src/Formats/registerWithNamesAndTypes.h b/src/Formats/registerWithNamesAndTypes.h index 50a0eee9616..1bcef5443f5 100644 --- a/src/Formats/registerWithNamesAndTypes.h +++ b/src/Formats/registerWithNamesAndTypes.h @@ -2,10 +2,10 @@ #include #include -#include namespace DB { +class FormatFactory; using RegisterWithNamesAndTypesFunc = std::function; void registerWithNamesAndTypes(const std::string & base_format_name, RegisterWithNamesAndTypesFunc register_func); diff --git a/src/Functions/CastOverloadResolver.cpp b/src/Functions/CastOverloadResolver.cpp index 6cb4d492fd8..e6888d9e280 100644 --- a/src/Functions/CastOverloadResolver.cpp +++ b/src/Functions/CastOverloadResolver.cpp @@ -12,6 +12,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool cast_keep_nullable; +} namespace ErrorCodes { @@ -76,7 +80,8 @@ public: else { const auto & settings_ref = context->getSettingsRef(); - return std::make_unique(context, cast_type, internal, diagnostic, settings_ref.cast_keep_nullable, DataTypeValidationSettings(settings_ref)); + return std::make_unique( + context, cast_type, internal, diagnostic, settings_ref[Setting::cast_keep_nullable], DataTypeValidationSettings(settings_ref)); } } diff --git a/src/Functions/FunctionCustomWeekToDateOrDate32.h b/src/Functions/FunctionCustomWeekToDateOrDate32.h index 6ae8c847b65..2d465d33d7b 100644 --- a/src/Functions/FunctionCustomWeekToDateOrDate32.h +++ b/src/Functions/FunctionCustomWeekToDateOrDate32.h @@ -4,6 +4,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_extended_results_for_datetime_functions; +} namespace ErrorCodes { @@ -23,7 +27,7 @@ public: } explicit FunctionCustomWeekToDateOrDate32(ContextPtr context) - : enable_extended_results_for_datetime_functions(context->getSettingsRef().enable_extended_results_for_datetime_functions) + : enable_extended_results_for_datetime_functions(context->getSettingsRef()[Setting::enable_extended_results_for_datetime_functions]) { } diff --git a/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h index 28caef36879..c1a2c9c08f5 100644 --- a/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h +++ b/src/Functions/FunctionDateOrDateTimeToDateOrDate32.h @@ -4,6 +4,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_extended_results_for_datetime_functions; +} namespace ErrorCodes { @@ -23,7 +27,7 @@ public: } explicit FunctionDateOrDateTimeToDateOrDate32(ContextPtr context_) - : enable_extended_results_for_datetime_functions(context_->getSettingsRef().enable_extended_results_for_datetime_functions) + : enable_extended_results_for_datetime_functions(context_->getSettingsRef()[Setting::enable_extended_results_for_datetime_functions]) { } diff --git a/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h b/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h index d5a1f28b7cd..bde4bbfa322 100644 --- a/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h +++ b/src/Functions/FunctionDateOrDateTimeToDateTimeOrDateTime64.h @@ -5,6 +5,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_extended_results_for_datetime_functions; +} namespace ErrorCodes { @@ -24,7 +28,7 @@ public: } explicit FunctionDateOrDateTimeToDateTimeOrDateTime64(ContextPtr context_) - : enable_extended_results_for_datetime_functions(context_->getSettingsRef().enable_extended_results_for_datetime_functions) + : enable_extended_results_for_datetime_functions(context_->getSettingsRef()[Setting::enable_extended_results_for_datetime_functions]) { } diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp index 501cf6e725c..70af9f881da 100644 --- a/src/Functions/FunctionFactory.cpp +++ b/src/Functions/FunctionFactory.cpp @@ -15,6 +15,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -133,7 +137,7 @@ FunctionOverloadResolverPtr FunctionFactory::tryGetImpl( if (CurrentThread::isInitialized()) { auto query_context = CurrentThread::get().getQueryContext(); - if (query_context && query_context->getSettingsRef().log_queries) + if (query_context && query_context->getSettingsRef()[Setting::log_queries]) query_context->addQueryFactoriesInfo(Context::QueryLogFactories::Function, name); } diff --git a/src/Functions/FunctionGenerateRandomStructure.cpp b/src/Functions/FunctionGenerateRandomStructure.cpp index 2bead8737fd..66a91d76e17 100644 --- a/src/Functions/FunctionGenerateRandomStructure.cpp +++ b/src/Functions/FunctionGenerateRandomStructure.cpp @@ -16,6 +16,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_suspicious_low_cardinality_types; +} namespace ErrorCodes { @@ -357,7 +361,7 @@ namespace FunctionPtr FunctionGenerateRandomStructure::create(DB::ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_suspicious_low_cardinality_types.value); + return std::make_shared(context->getSettingsRef()[Setting::allow_suspicious_low_cardinality_types].value); } DataTypePtr FunctionGenerateRandomStructure::getReturnTypeImpl(const DataTypes & arguments) const @@ -424,7 +428,7 @@ String FunctionGenerateRandomStructure::generateRandomStructure(size_t seed, con pcg64 rng(seed); size_t number_of_columns = generateNumberOfColumns(rng); WriteBufferFromOwnString buf; - writeRandomStructure(rng, number_of_columns, buf, context->getSettingsRef().allow_suspicious_low_cardinality_types); + writeRandomStructure(rng, number_of_columns, buf, context->getSettingsRef()[Setting::allow_suspicious_low_cardinality_types]); return buf.str(); } diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index a67f3a977db..b3703ffb2ac 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -13,6 +13,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -188,7 +192,7 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp } auto return_type = storage_join->joinGetCheckAndGetReturnType(data_types, attr_name, or_null || storage_join->useNulls()); - auto table_lock = storage_join->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock = storage_join->lockForShare(getContext()->getInitialQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); if (storage_join->useNulls()) return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 4721f858f5c..47fdf5eba27 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -29,6 +29,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_simdjson; + extern const SettingsBool function_json_value_return_type_allow_complex; + extern const SettingsBool function_json_value_return_type_allow_nullable; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -209,11 +218,11 @@ class FunctionSQLJSON : public IFunction public: static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } explicit FunctionSQLJSON(ContextPtr context_) - : max_parser_depth(context_->getSettingsRef().max_parser_depth), - max_parser_backtracks(context_->getSettingsRef().max_parser_backtracks), - allow_simdjson(context_->getSettingsRef().allow_simdjson), - function_json_value_return_type_allow_complex(context_->getSettingsRef().function_json_value_return_type_allow_complex), - function_json_value_return_type_allow_nullable(context_->getSettingsRef().function_json_value_return_type_allow_nullable) + : max_parser_depth(context_->getSettingsRef()[Setting::max_parser_depth]), + max_parser_backtracks(context_->getSettingsRef()[Setting::max_parser_backtracks]), + allow_simdjson(context_->getSettingsRef()[Setting::allow_simdjson]), + function_json_value_return_type_allow_complex(context_->getSettingsRef()[Setting::function_json_value_return_type_allow_complex]), + function_json_value_return_type_allow_nullable(context_->getSettingsRef()[Setting::function_json_value_return_type_allow_nullable]) { } diff --git a/src/Functions/FunctionTokens.h b/src/Functions/FunctionTokens.h index b6d8e9ee589..ceb2669056f 100644 --- a/src/Functions/FunctionTokens.h +++ b/src/Functions/FunctionTokens.h @@ -22,6 +22,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool splitby_max_substrings_includes_remaining_string; +} namespace ErrorCodes { @@ -64,7 +68,7 @@ public: explicit FunctionTokens(ContextPtr context) { const Settings & settings = context->getSettingsRef(); - max_substrings_includes_remaining_string = settings.splitby_max_substrings_includes_remaining_string; + max_substrings_includes_remaining_string = settings[Setting::splitby_max_substrings_includes_remaining_string]; } String getName() const override { return name; } diff --git a/src/Functions/FunctionUnixTimestamp64.cpp b/src/Functions/FunctionUnixTimestamp64.cpp index a22c035a0ba..75375e1fd7f 100644 --- a/src/Functions/FunctionUnixTimestamp64.cpp +++ b/src/Functions/FunctionUnixTimestamp64.cpp @@ -3,11 +3,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_nonconst_timezone_arguments; +} FunctionFromUnixTimestamp64::FunctionFromUnixTimestamp64(size_t target_scale_, const char * name_, ContextPtr context) : target_scale(target_scale_) , name(name_) - , allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + , allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} } diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index 0416df8f462..8c7828a0830 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -35,6 +35,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool cast_ipv4_ipv6_default_on_conversion_error; +} namespace ErrorCodes { @@ -267,7 +271,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionIPv6StringToNum(ContextPtr context) - : cast_ipv4_ipv6_default_on_conversion_error(context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error) + : cast_ipv4_ipv6_default_on_conversion_error(context->getSettingsRef()[Setting::cast_ipv4_ipv6_default_on_conversion_error]) { } @@ -434,7 +438,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionIPv4StringToNum(ContextPtr context) - : cast_ipv4_ipv6_default_on_conversion_error(context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error) + : cast_ipv4_ipv6_default_on_conversion_error(context->getSettingsRef()[Setting::cast_ipv4_ipv6_default_on_conversion_error]) { } diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 271daa99d0c..ed13e581759 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -75,6 +75,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool cast_ipv4_ipv6_default_on_conversion_error; + extern const SettingsBool cast_string_to_dynamic_use_inference; + extern const SettingsDateTimeOverflowBehavior date_time_overflow_behavior; + extern const SettingsBool input_format_ipv4_default_on_conversion_error; + extern const SettingsBool input_format_ipv6_default_on_conversion_error; + extern const SettingsBool precise_float_parsing; +} namespace ErrorCodes { @@ -967,7 +976,7 @@ struct ConvertThroughParsing const DB::ContextPtr query_context = DB::CurrentThread::get().getQueryContext(); if (query_context) - precise_float_parsing = query_context->getSettingsRef().precise_float_parsing; + precise_float_parsing = query_context->getSettingsRef()[Setting::precise_float_parsing]; } for (size_t i = 0; i < size; ++i) @@ -2217,7 +2226,7 @@ private: FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior = default_date_time_overflow_behavior; if (context) - date_time_overflow_behavior = context->getSettingsRef().date_time_overflow_behavior.value; + date_time_overflow_behavior = context->getSettingsRef()[Setting::date_time_overflow_behavior].value; auto call = [&](const auto & types, BehaviourOnErrorFromString from_string_tag) -> bool { @@ -2354,7 +2363,7 @@ private: bool cast_ipv4_ipv6_default_on_conversion_error = false; if constexpr (is_any_of) { - if (context && (cast_ipv4_ipv6_default_on_conversion_error = context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error)) + if (context && (cast_ipv4_ipv6_default_on_conversion_error = context->getSettingsRef()[Setting::cast_ipv4_ipv6_default_on_conversion_error])) done = callOnIndexAndDataType(from_type->getTypeId(), call, BehaviourOnErrorFromString::ConvertReturnZeroOnErrorTag); } @@ -3226,7 +3235,7 @@ private: FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior = default_date_time_overflow_behavior; if (context) - date_time_overflow_behavior = context->getSettingsRef().date_time_overflow_behavior; + date_time_overflow_behavior = context->getSettingsRef()[Setting::date_time_overflow_behavior]; if (requested_result_is_nullable && checkAndGetDataType(from_type.get())) { @@ -4570,7 +4579,7 @@ private: if (const auto * variant_type = typeid_cast(from_type.get())) return createVariantToDynamicWrapper(*variant_type, dynamic_type); - if (context && context->getSettingsRef().cast_string_to_dynamic_use_inference && isStringOrFixedString(removeNullable(removeLowCardinality(from_type)))) + if (context && context->getSettingsRef()[Setting::cast_string_to_dynamic_use_inference] && isStringOrFixedString(removeNullable(removeLowCardinality(from_type)))) return createStringToDynamicThroughParsingWrapper(); /// First, cast column to Variant with 2 variants - the type of the column we cast and shared variant type. @@ -5069,9 +5078,9 @@ private: return false; }; - bool cast_ipv4_ipv6_default_on_conversion_error_value = context && context->getSettingsRef().cast_ipv4_ipv6_default_on_conversion_error; - bool input_format_ipv4_default_on_conversion_error_value = context && context->getSettingsRef().input_format_ipv4_default_on_conversion_error; - bool input_format_ipv6_default_on_conversion_error_value = context && context->getSettingsRef().input_format_ipv6_default_on_conversion_error; + bool cast_ipv4_ipv6_default_on_conversion_error_value = context && context->getSettingsRef()[Setting::cast_ipv4_ipv6_default_on_conversion_error]; + bool input_format_ipv4_default_on_conversion_error_value = context && context->getSettingsRef()[Setting::input_format_ipv4_default_on_conversion_error]; + bool input_format_ipv6_default_on_conversion_error_value = context && context->getSettingsRef()[Setting::input_format_ipv6_default_on_conversion_error]; auto make_custom_serialization_wrapper = [&, cast_ipv4_ipv6_default_on_conversion_error_value, input_format_ipv4_default_on_conversion_error_value, input_format_ipv6_default_on_conversion_error_value](const auto & types) -> bool { diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index e6892642d56..4106ade6dee 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -44,6 +44,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_simdjson; +} namespace ErrorCodes { @@ -531,7 +535,7 @@ public: for (const auto & argument : arguments) argument_types.emplace_back(argument.type); return std::make_unique>( - null_presence, getContext()->getSettingsRef().allow_simdjson, argument_types, return_type, json_return_type, getFormatSettings(getContext())); + null_presence, getContext()->getSettingsRef()[Setting::allow_simdjson], argument_types, return_type, json_return_type, getFormatSettings(getContext())); } }; diff --git a/src/Functions/FunctionsLanguageClassification.cpp b/src/Functions/FunctionsLanguageClassification.cpp index 410eea0f437..9936a74db9b 100644 --- a/src/Functions/FunctionsLanguageClassification.cpp +++ b/src/Functions/FunctionsLanguageClassification.cpp @@ -18,6 +18,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_nlp_functions; +} + /* Determine language of Unicode UTF-8 text. * Uses the cld2 library https://github.com/CLD2Owners/cld2 */ @@ -114,7 +119,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_experimental_nlp_functions) + if (!context->getSettingsRef()[Setting::allow_experimental_nlp_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Natural language processing function '{}' is experimental. " "Set `allow_experimental_nlp_functions` setting to enable it", name); diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 8346380c35d..1a202236534 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -18,6 +18,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_hyperscan; + extern const SettingsUInt64 max_hyperscan_regexp_length; + extern const SettingsUInt64 max_hyperscan_regexp_total_length; + extern const SettingsBool reject_expensive_hyperscan_regexps; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -40,7 +48,7 @@ public: static FunctionPtr create(ContextPtr context) { const auto & settings = context->getSettingsRef(); - return std::make_shared(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length, settings.reject_expensive_hyperscan_regexps); + return std::make_shared(settings[Setting::allow_hyperscan], settings[Setting::max_hyperscan_regexp_length], settings[Setting::max_hyperscan_regexp_total_length], settings[Setting::reject_expensive_hyperscan_regexps]); } FunctionsMultiStringFuzzySearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_, bool reject_expensive_hyperscan_regexps_) diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 6bcc8581a38..11d9f894d8b 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -17,6 +17,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_hyperscan; + extern const SettingsUInt64 max_hyperscan_regexp_length; + extern const SettingsUInt64 max_hyperscan_regexp_total_length; + extern const SettingsBool reject_expensive_hyperscan_regexps; +} + /** * multiMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) * multiMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n]) @@ -54,7 +62,7 @@ public: static FunctionPtr create(ContextPtr context) { const auto & settings = context->getSettingsRef(); - return std::make_shared(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length, settings.reject_expensive_hyperscan_regexps); + return std::make_shared(settings[Setting::allow_hyperscan], settings[Setting::max_hyperscan_regexp_length], settings[Setting::max_hyperscan_regexp_total_length], settings[Setting::reject_expensive_hyperscan_regexps]); } FunctionsMultiStringSearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_, bool reject_expensive_hyperscan_regexps_) diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index 7ec0076e395..66b85c543cc 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -17,6 +17,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool function_locate_has_mysql_compatible_argument_order; +} + /** Search and replace functions in strings: * position(haystack, needle) - the normal search for a substring in a string, returns the position (in bytes) of the found substring starting with 1, or 0 if no substring is found. * positionUTF8(haystack, needle) - the same, but the position is calculated at code points, provided that the string is encoded in UTF-8. @@ -99,7 +104,7 @@ public: { if constexpr (haystack_needle_order_is_configurable == HaystackNeedleOrderIsConfigurable::Yes) { - if (context->getSettingsRef().function_locate_has_mysql_compatible_argument_order) + if (context->getSettingsRef()[Setting::function_locate_has_mysql_compatible_argument_order]) argument_order = ArgumentOrder::NeedleHaystack; } } diff --git a/src/Functions/FunctionsTextClassification.h b/src/Functions/FunctionsTextClassification.h index d5cba690f81..fa55c932a9d 100644 --- a/src/Functions/FunctionsTextClassification.h +++ b/src/Functions/FunctionsTextClassification.h @@ -12,6 +12,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_nlp_functions; +} + /// Functions for text classification with different result types namespace ErrorCodes @@ -29,7 +34,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_experimental_nlp_functions) + if (!context->getSettingsRef()[Setting::allow_experimental_nlp_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Natural language processing function '{}' is experimental. " "Set `allow_experimental_nlp_functions` setting to enable it", name); @@ -78,7 +83,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_experimental_nlp_functions) + if (!context->getSettingsRef()[Setting::allow_experimental_nlp_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Natural language processing function '{}' is experimental. " "Set `allow_experimental_nlp_functions` setting to enable it", name); diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h index 48daf13ba68..6013f818f87 100644 --- a/src/Functions/PerformanceAdaptors.h +++ b/src/Functions/PerformanceAdaptors.h @@ -7,6 +7,8 @@ #include #include +#include + #include #include @@ -16,6 +18,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsString function_implementation; +} namespace ErrorCodes { @@ -249,7 +255,7 @@ public: if (isArchSupported(Arch)) { // TODO(dakovalkov): make this option better. - const auto & choose_impl = getContext()->getSettingsRef().function_implementation.value; + const auto & choose_impl = getContext()->getSettingsRef()[Setting::function_implementation].value; if (choose_impl.empty() || choose_impl == detail::getImplementationTag(Arch)) { implementations.emplace_back(std::make_shared(std::forward(args)...)); diff --git a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp index a20e1807487..eb16863fac9 100644 --- a/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp +++ b/src/Functions/UserDefined/ExternalUserDefinedExecutableFunctionsLoader.cpp @@ -14,6 +14,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -185,7 +189,7 @@ ExternalLoader::LoadableMutablePtr ExternalUserDefinedExecutableFunctionsLoader: pool_size = config.getUInt64(key_in_config + ".pool_size", 16); max_command_execution_time = config.getUInt64(key_in_config + ".max_command_execution_time", 10); - size_t max_execution_time_seconds = static_cast(getContext()->getSettingsRef().max_execution_time.totalSeconds()); + size_t max_execution_time_seconds = static_cast(getContext()->getSettingsRef()[Setting::max_execution_time].totalSeconds()); if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; } diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp index d0bc812f91d..3dee7e5a61b 100644 --- a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp @@ -19,6 +19,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSetOperationMode union_default_mode; +} namespace ErrorCodes { @@ -89,7 +93,7 @@ namespace res.if_not_exists = false; res.or_replace = false; FunctionNameNormalizer::visit(res.function_core.get()); - NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef().union_default_mode}; + NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef()[Setting::union_default_mode]}; NormalizeSelectWithUnionQueryVisitor{data}.visit(res.function_core); return ptr; } diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp index da79e17f18a..ea5a9f0417c 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp @@ -18,6 +18,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -129,7 +134,8 @@ restoreUserDefinedSQLObjects(RestorerFromBackup & restorer, const String & data_ statement_def.data() + statement_def.size(), "in file " + filepath + " from backup " + backup->getNameForLogging(), 0, - context->getSettingsRef().max_parser_depth, context->getSettingsRef().max_parser_backtracks); + context->getSettingsRef()[Setting::max_parser_depth], + context->getSettingsRef()[Setting::max_parser_backtracks]); break; } } diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp index 8910b45e79d..a3af0b66365 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp @@ -32,6 +32,12 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool fsync_metadata; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -93,8 +99,8 @@ ASTPtr UserDefinedSQLObjectsDiskStorage::tryLoadObject(UserDefinedSQLObjectType object_create_query.data() + object_create_query.size(), "", 0, - global_context->getSettingsRef().max_parser_depth, - global_context->getSettingsRef().max_parser_backtracks); + global_context->getSettingsRef()[Setting::max_parser_depth], + global_context->getSettingsRef()[Setting::max_parser_backtracks]); return ast; } } @@ -214,7 +220,7 @@ bool UserDefinedSQLObjectsDiskStorage::storeObjectImpl( WriteBufferFromFile out(temp_file_path, create_statement.size()); writeString(create_statement, out); out.next(); - if (settings.fsync_metadata) + if (settings[Setting::fsync_metadata]) out.sync(); out.close(); diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp index 225e919301d..1df57f76f23 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp @@ -10,6 +10,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSetOperationMode union_default_mode; +} namespace ErrorCodes { @@ -27,7 +31,7 @@ ASTPtr normalizeCreateFunctionQuery(const IAST & create_function_query, const Co res.if_not_exists = false; res.or_replace = false; FunctionNameNormalizer::visit(res.function_core.get()); - NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef().union_default_mode}; + NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef()[Setting::union_default_mode]}; NormalizeSelectWithUnionQueryVisitor{data}.visit(res.function_core); return ptr; } diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp index 5c039ecbd60..e7e3d5932f4 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp @@ -18,6 +18,11 @@ namespace DB { +namespace Setting +{ +extern const SettingsUInt64 max_parser_backtracks; +extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -312,8 +317,8 @@ ASTPtr UserDefinedSQLObjectsZooKeeperStorage::parseObjectData(const String & obj object_data.data() + object_data.size(), "", 0, - global_context->getSettingsRef().max_parser_depth, - global_context->getSettingsRef().max_parser_backtracks); + global_context->getSettingsRef()[Setting::max_parser_depth], + global_context->getSettingsRef()[Setting::max_parser_backtracks]); return ast; } } diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index d2aedd57f99..db961ada0f9 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool use_variant_as_common_type; +} /// array(c1, c2, ...) - create an array. class FunctionArray : public IFunction @@ -25,7 +31,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type); + return std::make_shared(context->getSettingsRef()[Setting::allow_experimental_variant_type] && context->getSettingsRef()[Setting::use_variant_as_common_type]); } bool useDefaultImplementationForNulls() const override { return false; } diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index b8b03ab5e74..f387d30c319 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -18,6 +18,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 function_range_max_elements_in_block; +} namespace ErrorCodes { @@ -41,7 +45,7 @@ public: const size_t max_elements; static FunctionPtr create(ContextPtr context_) { return std::make_shared(std::move(context_)); } - explicit FunctionRange(ContextPtr context) : max_elements(context->getSettingsRef().function_range_max_elements_in_block) {} + explicit FunctionRange(ContextPtr context) : max_elements(context->getSettingsRef()[Setting::function_range_max_elements_in_block]) { } private: String getName() const override { return name; } diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp index 1ece17ca1e3..c52266c8832 100644 --- a/src/Functions/castOrDefault.cpp +++ b/src/Functions/castOrDefault.cpp @@ -24,6 +24,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool cast_keep_nullable; +} namespace ErrorCodes { @@ -42,10 +46,7 @@ public: return std::make_shared(context); } - explicit FunctionCastOrDefault(ContextPtr context_) - : keep_nullable(context_->getSettingsRef().cast_keep_nullable) - { - } + explicit FunctionCastOrDefault(ContextPtr context_) : keep_nullable(context_->getSettingsRef()[Setting::cast_keep_nullable]) { } String getName() const override { return name; } diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index 7732855b211..06f03a2c26a 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -17,6 +17,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 regexp_max_matches_per_row; +} namespace ErrorCodes { @@ -155,7 +159,7 @@ public: else { /// Additional limit to fail fast on supposedly incorrect usage. - const auto max_matches_per_row = context->getSettingsRef().regexp_max_matches_per_row; + const auto max_matches_per_row = context->getSettingsRef()[Setting::regexp_max_matches_per_row]; PODArray all_matches; /// Number of times RE matched on each row of haystack column. diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index f33b7849a43..ae258646015 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -31,6 +31,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool formatdatetime_format_without_leading_zeros; + extern const SettingsBool formatdatetime_f_prints_single_zero; + extern const SettingsBool formatdatetime_parsedatetime_m_is_month_name; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -783,9 +790,9 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionFormatDateTimeImpl(ContextPtr context) - : mysql_M_is_month_name(context->getSettingsRef().formatdatetime_parsedatetime_m_is_month_name) - , mysql_f_prints_single_zero(context->getSettingsRef().formatdatetime_f_prints_single_zero) - , mysql_format_ckl_without_leading_zeros(context->getSettingsRef().formatdatetime_format_without_leading_zeros) + : mysql_M_is_month_name(context->getSettingsRef()[Setting::formatdatetime_parsedatetime_m_is_month_name]) + , mysql_f_prints_single_zero(context->getSettingsRef()[Setting::formatdatetime_f_prints_single_zero]) + , mysql_format_ckl_without_leading_zeros(context->getSettingsRef()[Setting::formatdatetime_format_without_leading_zeros]) { } diff --git a/src/Functions/formatQuery.cpp b/src/Functions/formatQuery.cpp index be633bdfe37..42c7a23dbfa 100644 --- a/src/Functions/formatQuery.cpp +++ b/src/Functions/formatQuery.cpp @@ -12,6 +12,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsBool print_pretty_type_names; +} namespace ErrorCodes { @@ -40,10 +47,10 @@ public: : name(name_), output_formatting(output_formatting_), error_handling(error_handling_) { const Settings & settings = context->getSettingsRef(); - max_query_size = settings.max_query_size; - max_parser_depth = settings.max_parser_depth; - max_parser_backtracks = settings.max_parser_backtracks; - print_pretty_type_names = settings.print_pretty_type_names; + max_query_size = settings[Setting::max_query_size]; + max_parser_depth = settings[Setting::max_parser_depth]; + max_parser_backtracks = settings[Setting::max_parser_backtracks]; + print_pretty_type_names = settings[Setting::print_pretty_type_names]; } String getName() const override { return name; } diff --git a/src/Functions/getClientHTTPHeader.cpp b/src/Functions/getClientHTTPHeader.cpp index 50a6275fc82..ee6d8715591 100644 --- a/src/Functions/getClientHTTPHeader.cpp +++ b/src/Functions/getClientHTTPHeader.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_get_client_http_header; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -25,7 +30,7 @@ public: explicit FunctionGetClientHTTPHeader(ContextPtr context_) : WithContext(context_) { - if (!getContext()->getSettingsRef().allow_get_client_http_header) + if (!getContext()->getSettingsRef()[Setting::allow_get_client_http_header]) throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "The function getClientHTTPHeader requires setting `allow_get_client_http_header` to be enabled."); } diff --git a/src/Functions/getScalar.cpp b/src/Functions/getScalar.cpp index 5131dca962e..001fc4bad47 100644 --- a/src/Functions/getScalar.cpp +++ b/src/Functions/getScalar.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index 1bd71f19f76..0aca78f0d3f 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -13,6 +13,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool geo_distance_returns_float64_on_float64_arguments; +} namespace ErrorCodes { @@ -229,7 +233,7 @@ class FunctionGeoDistance : public IFunction public: explicit FunctionGeoDistance(ContextPtr context) { - always_float32 = !context->getSettingsRef().geo_distance_returns_float64_on_float64_arguments; + always_float32 = !context->getSettingsRef()[Setting::geo_distance_returns_float64_on_float64_arguments]; } private: diff --git a/src/Functions/hilbertDecode.cpp b/src/Functions/hilbertDecode.cpp index df7f98f56ac..8ee845da18f 100644 --- a/src/Functions/hilbertDecode.cpp +++ b/src/Functions/hilbertDecode.cpp @@ -1,8 +1,7 @@ -#include -#include -#include -#include "hilbertDecode2DLUT.h" #include +#include +#include +#include "hilbertDecode2DLUT.h" namespace DB diff --git a/src/Functions/hilbertEncode.cpp b/src/Functions/hilbertEncode.cpp index a4e3cc59b76..b982512ea0b 100644 --- a/src/Functions/hilbertEncode.cpp +++ b/src/Functions/hilbertEncode.cpp @@ -1,9 +1,8 @@ -#include "hilbertEncode2DLUT.h" -#include -#include #include #include #include +#include +#include "hilbertEncode2DLUT.h" namespace DB diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index 8829b3c4ff1..73fd9f13db7 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -35,6 +35,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool use_variant_as_common_type; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -261,7 +267,7 @@ public: static constexpr auto name = "if"; static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type); + return std::make_shared(context->getSettingsRef()[Setting::allow_experimental_variant_type] && context->getSettingsRef()[Setting::use_variant_as_common_type]); } explicit FunctionIf(bool use_variant_when_no_common_type_ = false) : FunctionIfBase(), use_variant_when_no_common_type(use_variant_when_no_common_type_) {} diff --git a/src/Functions/isNotNull.cpp b/src/Functions/isNotNull.cpp index a48ace4243f..3fe046d7d1b 100644 --- a/src/Functions/isNotNull.cpp +++ b/src/Functions/isNotNull.cpp @@ -1,17 +1,23 @@ +#include #include #include #include -#include #include +#include #include #include #include #include -#include +#include #include namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} + namespace { @@ -22,7 +28,10 @@ class FunctionIsNotNull : public IFunction public: static constexpr auto name = "isNotNull"; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context->getSettingsRef().allow_experimental_analyzer); } + static FunctionPtr create(ContextPtr context) + { + return std::make_shared(context->getSettingsRef()[Setting::allow_experimental_analyzer]); + } explicit FunctionIsNotNull(bool use_analyzer_) : use_analyzer(use_analyzer_) {} diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp index 6d799a4ee4e..52e26777f7d 100644 --- a/src/Functions/isNull.cpp +++ b/src/Functions/isNull.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} + namespace { @@ -25,7 +30,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_experimental_analyzer); + return std::make_shared(context->getSettingsRef()[Setting::allow_experimental_analyzer]); } explicit FunctionIsNull(bool use_analyzer_) : use_analyzer(use_analyzer_) {} diff --git a/src/Functions/isNullable.cpp b/src/Functions/isNullable.cpp index d21ac9cf07c..7928749375c 100644 --- a/src/Functions/isNullable.cpp +++ b/src/Functions/isNullable.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} + namespace { @@ -18,7 +23,7 @@ public: static constexpr auto name = "isNullable"; static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().allow_experimental_analyzer); + return std::make_shared(context->getSettingsRef()[Setting::allow_experimental_analyzer]); } explicit FunctionIsNullable(bool use_analyzer_) : use_analyzer(use_analyzer_) {} diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp index cc9e57ac186..7197b5f75d5 100644 --- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp +++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp @@ -17,6 +17,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 extract_key_value_pairs_max_pairs_per_row; +} + template class ExtractKeyValuePairs : public IFunction { @@ -44,11 +49,11 @@ class ExtractKeyValuePairs : public IFunction builder.withQuotingCharacter(parsed_arguments.quoting_character.value()); } - bool is_number_of_pairs_unlimited = context->getSettingsRef().extract_key_value_pairs_max_pairs_per_row == 0; + bool is_number_of_pairs_unlimited = context->getSettingsRef()[Setting::extract_key_value_pairs_max_pairs_per_row] == 0; if (!is_number_of_pairs_unlimited) { - builder.withMaxNumberOfPairs(context->getSettingsRef().extract_key_value_pairs_max_pairs_per_row); + builder.withMaxNumberOfPairs(context->getSettingsRef()[Setting::extract_key_value_pairs_max_pairs_per_row]); } return builder.build(); diff --git a/src/Functions/lemmatize.cpp b/src/Functions/lemmatize.cpp index 76e5ed7432e..7c9b7d8105d 100644 --- a/src/Functions/lemmatize.cpp +++ b/src/Functions/lemmatize.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_nlp_functions; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -61,7 +66,7 @@ public: static constexpr auto name = "lemmatize"; static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_experimental_nlp_functions) + if (!context->getSettingsRef()[Setting::allow_experimental_nlp_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Natural language processing function '{}' is experimental. " "Set `allow_experimental_nlp_functions` setting to enable it", name); diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 14672cfd568..1f97fc5cca7 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -19,6 +19,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool use_variant_as_common_type; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -40,7 +46,7 @@ public: explicit FunctionMap(ContextPtr context_) : context(context_) , use_variant_as_common_type( - context->getSettingsRef().allow_experimental_variant_type && context->getSettingsRef().use_variant_as_common_type) + context->getSettingsRef()[Setting::allow_experimental_variant_type] && context->getSettingsRef()[Setting::use_variant_as_common_type]) , function_array(FunctionFactory::instance().get("array", context)) , function_map_from_arrays(FunctionFactory::instance().get("mapFromArrays", context)) { diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 14b8b70b22c..088d4d0555b 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -24,6 +24,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_execute_multiif_columnar; + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool use_variant_as_common_type; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -52,7 +59,8 @@ public: static FunctionPtr create(ContextPtr context_) { const auto & settings = context_->getSettingsRef(); - return std::make_shared(settings.allow_execute_multiif_columnar, settings.allow_experimental_variant_type, settings.use_variant_as_common_type); + return std::make_shared( + settings[Setting::allow_execute_multiif_columnar], settings[Setting::allow_experimental_variant_type], settings[Setting::use_variant_as_common_type]); } explicit FunctionMultiIf(bool allow_execute_multiif_columnar_, bool allow_experimental_variant_type_, bool use_variant_as_common_type_) diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index 49b8980e7e2..74164218b15 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_error_prone_window_functions; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -37,7 +42,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_deprecated_error_prone_window_functions) + if (!context->getSettingsRef()[Setting::allow_deprecated_error_prone_window_functions]) throw Exception( ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated since its usage is error-prone (see docs)." diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp index 7b2150e3534..85bb1a37d98 100644 --- a/src/Functions/now.cpp +++ b/src/Functions/now.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_nonconst_timezone_arguments; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -91,7 +96,7 @@ public: size_t getNumberOfArguments() const override { return 0; } static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } explicit NowOverloadResolver(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index 9786a0c9f39..76dea69c460 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_nonconst_timezone_arguments; +} + namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -119,7 +124,7 @@ public: size_t getNumberOfArguments() const override { return 0; } static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } explicit Now64OverloadResolver(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Functions/nowInBlock.cpp b/src/Functions/nowInBlock.cpp index 291e43bdce5..b9c2ee97298 100644 --- a/src/Functions/nowInBlock.cpp +++ b/src/Functions/nowInBlock.cpp @@ -10,6 +10,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_nonconst_timezone_arguments; +} namespace ErrorCodes { @@ -32,7 +36,7 @@ public: return std::make_shared(context); } explicit FunctionNowInBlock(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} String getName() const override diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 7ca10677be7..2dd6db94cfb 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -24,6 +24,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool formatdatetime_parsedatetime_m_is_month_name; + extern const SettingsBool parsedatetime_parse_without_leading_zeros; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -565,8 +571,8 @@ namespace static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionParseDateTimeImpl(ContextPtr context) - : mysql_M_is_month_name(context->getSettingsRef().formatdatetime_parsedatetime_m_is_month_name) - , mysql_parse_ckl_without_leading_zeros(context->getSettingsRef().parsedatetime_parse_without_leading_zeros) + : mysql_M_is_month_name(context->getSettingsRef()[Setting::formatdatetime_parsedatetime_m_is_month_name]) + , mysql_parse_ckl_without_leading_zeros(context->getSettingsRef()[Setting::parsedatetime_parse_without_leading_zeros]) { } diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 32d5905a513..909fc7a1793 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -33,6 +33,11 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool validate_polygons; +} + namespace ErrorCodes { extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; @@ -60,8 +65,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared>( - context->getSettingsRef().validate_polygons); + return std::make_shared>(context->getSettingsRef()[Setting::validate_polygons]); } String getName() const override diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index c94cbe67216..c5821ca0d24 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_error_prone_window_functions; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -40,7 +45,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_deprecated_error_prone_window_functions) + if (!context->getSettingsRef()[Setting::allow_deprecated_error_prone_window_functions]) throw Exception( ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated since its usage is error-prone (see docs)." diff --git a/src/Functions/runningDifference.h b/src/Functions/runningDifference.h index d51ea805f3c..e8143c55c27 100644 --- a/src/Functions/runningDifference.h +++ b/src/Functions/runningDifference.h @@ -17,6 +17,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_error_prone_window_functions; +} namespace ErrorCodes { @@ -140,7 +144,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_deprecated_error_prone_window_functions) + if (!context->getSettingsRef()[Setting::allow_deprecated_error_prone_window_functions]) throw Exception( ErrorCodes::DEPRECATED_FUNCTION, "Function {} is deprecated since its usage is error-prone (see docs)." diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index fe999d66701..a424069076b 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/sleep.h b/src/Functions/sleep.h index b6e4b36ee64..69114884f96 100644 --- a/src/Functions/sleep.h +++ b/src/Functions/sleep.h @@ -24,6 +24,10 @@ extern const Event SleepFunctionElapsedMicroseconds; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 function_sleep_max_microseconds_per_block; +} namespace ErrorCodes { @@ -59,8 +63,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared>( - context->getSettingsRef().function_sleep_max_microseconds_per_block, - context->getProcessListElementSafe()); + context->getSettingsRef()[Setting::function_sleep_max_microseconds_per_block], context->getProcessListElementSafe()); } FunctionSleep(UInt64 max_microseconds_, QueryStatusPtr query_status_) diff --git a/src/Functions/snowflake.cpp b/src/Functions/snowflake.cpp index 0d4d65ee4b0..227d079ea54 100644 --- a/src/Functions/snowflake.cpp +++ b/src/Functions/snowflake.cpp @@ -19,6 +19,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_snowflake_conversion_functions; + extern const SettingsBool allow_nonconst_timezone_arguments; +} namespace ErrorCodes { @@ -52,7 +57,7 @@ public: } explicit FunctionDateTimeToSnowflake(ContextPtr context) - : allow_deprecated_snowflake_conversion_functions(context->getSettingsRef().allow_deprecated_snowflake_conversion_functions) + : allow_deprecated_snowflake_conversion_functions(context->getSettingsRef()[Setting::allow_deprecated_snowflake_conversion_functions]) {} String getName() const override { return name; } @@ -104,8 +109,8 @@ public: } explicit FunctionSnowflakeToDateTime(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) - , allow_deprecated_snowflake_conversion_functions(context->getSettingsRef().allow_deprecated_snowflake_conversion_functions) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) + , allow_deprecated_snowflake_conversion_functions(context->getSettingsRef()[Setting::allow_deprecated_snowflake_conversion_functions]) {} String getName() const override { return name; } @@ -178,7 +183,7 @@ public: } explicit FunctionDateTime64ToSnowflake(ContextPtr context) - : allow_deprecated_snowflake_conversion_functions(context->getSettingsRef().allow_deprecated_snowflake_conversion_functions) + : allow_deprecated_snowflake_conversion_functions(context->getSettingsRef()[Setting::allow_deprecated_snowflake_conversion_functions]) {} String getName() const override { return name; } @@ -238,8 +243,8 @@ public: } explicit FunctionSnowflakeToDateTime64(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) - , allow_deprecated_snowflake_conversion_functions(context->getSettingsRef().allow_deprecated_snowflake_conversion_functions) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) + , allow_deprecated_snowflake_conversion_functions(context->getSettingsRef()[Setting::allow_deprecated_snowflake_conversion_functions]) {} String getName() const override { return name; } diff --git a/src/Functions/snowflakeIDToDateTime.cpp b/src/Functions/snowflakeIDToDateTime.cpp index 6c57c72fff1..178069d4816 100644 --- a/src/Functions/snowflakeIDToDateTime.cpp +++ b/src/Functions/snowflakeIDToDateTime.cpp @@ -15,6 +15,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_nonconst_timezone_arguments; +} namespace ErrorCodes { @@ -39,7 +43,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionSnowflakeIDToDateTime(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} String getName() const override { return name; } @@ -110,7 +114,7 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } explicit FunctionSnowflakeIDToDateTime64(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} String getName() const override { return name; } diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp index b3be40f4022..359be06a8ab 100644 --- a/src/Functions/stem.cpp +++ b/src/Functions/stem.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_nlp_functions; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -73,7 +78,7 @@ public: static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_experimental_nlp_functions) + if (!context->getSettingsRef()[Setting::allow_experimental_nlp_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Natural language processing function '{}' is experimental. " "Set `allow_experimental_nlp_functions` setting to enable it", name); diff --git a/src/Functions/synonyms.cpp b/src/Functions/synonyms.cpp index 18c1557115f..c85d4533793 100644 --- a/src/Functions/synonyms.cpp +++ b/src/Functions/synonyms.cpp @@ -19,6 +19,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_nlp_functions; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -32,7 +37,7 @@ public: static constexpr auto name = "synonyms"; static FunctionPtr create(ContextPtr context) { - if (!context->getSettingsRef().allow_experimental_nlp_functions) + if (!context->getSettingsRef()[Setting::allow_experimental_nlp_functions]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Natural language processing function '{}' is experimental. " "Set `allow_experimental_nlp_functions` setting to enable it", name); diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index e317c65c622..73ad313aeed 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_custom_error_code_in_throwif; +} + namespace ErrorCodes { extern const int ILLEGAL_COLUMN; @@ -31,7 +36,10 @@ public: static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } - explicit FunctionThrowIf(ContextPtr context_) : allow_custom_error_code_argument(context_->getSettingsRef().allow_custom_error_code_in_throwif) {} + explicit FunctionThrowIf(ContextPtr context_) + : allow_custom_error_code_argument(context_->getSettingsRef()[Setting::allow_custom_error_code_in_throwif]) + { + } String getName() const override { return name; } bool isVariadic() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index cbcdf9bc772..f6c1801e686 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_nonconst_timezone_arguments; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -89,7 +94,7 @@ public: size_t getNumberOfArguments() const override { return 2; } static FunctionOverloadResolverPtr create(ContextPtr context) { return std::make_unique(context); } explicit ToTimeZoneOverloadResolver(ContextPtr context) - : allow_nonconst_timezone_arguments(context->getSettingsRef().allow_nonconst_timezone_arguments) + : allow_nonconst_timezone_arguments(context->getSettingsRef()[Setting::allow_nonconst_timezone_arguments]) {} DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index abe02148d4a..3f1fe41ac4c 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool print_pretty_type_names; +} + namespace { @@ -25,7 +30,7 @@ public: static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->getSettingsRef().print_pretty_type_names); + return std::make_shared(context->getSettingsRef()[Setting::print_pretty_type_names]); } String getName() const override diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 84772f79d08..9b8532afcea 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -4,10 +4,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_named_columns_in_function_tuple; +} FunctionPtr FunctionTuple::create(DB::ContextPtr context) { - return std::make_shared(context->getSettingsRef().enable_named_columns_in_function_tuple); + return std::make_shared(context->getSettingsRef()[Setting::enable_named_columns_in_function_tuple]); } REGISTER_FUNCTION(Tuple) diff --git a/src/Functions/visibleWidth.cpp b/src/Functions/visibleWidth.cpp index 3e70418a456..f6b57aa59fa 100644 --- a/src/Functions/visibleWidth.cpp +++ b/src/Functions/visibleWidth.cpp @@ -12,6 +12,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 function_visible_width_behavior; +} namespace ErrorCodes { @@ -34,10 +38,7 @@ public: return std::make_shared(context); } - explicit FunctionVisibleWidth(ContextPtr context) - { - behavior = context->getSettingsRef().function_visible_width_behavior; - } + explicit FunctionVisibleWidth(ContextPtr context) { behavior = context->getSettingsRef()[Setting::function_visible_width_behavior]; } bool useDefaultImplementationForNulls() const override { return false; } ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const override { return {0}; } diff --git a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp index c10a7cd017a..72b8110fb75 100644 --- a/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp +++ b/src/IO/AzureBlobStorage/copyAzureBlobStorageFile.cpp @@ -2,12 +2,12 @@ #if USE_AZURE_BLOB_STORAGE +#include #include #include #include #include #include -#include #include #include #include diff --git a/src/IO/ConnectionTimeouts.cpp b/src/IO/ConnectionTimeouts.cpp index 662a2c067cf..ea160c135d8 100644 --- a/src/IO/ConnectionTimeouts.cpp +++ b/src/IO/ConnectionTimeouts.cpp @@ -1,12 +1,27 @@ #include #include #include -#include #include namespace DB { +namespace Setting +{ + extern const SettingsSeconds connect_timeout; + extern const SettingsSeconds send_timeout; + extern const SettingsSeconds receive_timeout; + extern const SettingsSeconds tcp_keep_alive_timeout; + extern const SettingsMilliseconds handshake_timeout_ms; + extern const SettingsMilliseconds hedged_connection_timeout_ms; + extern const SettingsMilliseconds receive_data_timeout_ms; + extern const SettingsMilliseconds connect_timeout_with_failover_ms; + extern const SettingsMilliseconds connect_timeout_with_failover_secure_ms; + extern const SettingsSeconds http_connection_timeout; + extern const SettingsSeconds http_send_timeout; + extern const SettingsSeconds http_receive_timeout; +} + Poco::Timespan ConnectionTimeouts::saturate(Poco::Timespan timespan, Poco::Timespan limit) { @@ -20,32 +35,32 @@ Poco::Timespan ConnectionTimeouts::saturate(Poco::Timespan timespan, Poco::Times ConnectionTimeouts ConnectionTimeouts::getTCPTimeoutsWithoutFailover(const Settings & settings) { return ConnectionTimeouts() - .withConnectionTimeout(settings.connect_timeout) - .withSendTimeout(settings.send_timeout) - .withReceiveTimeout(settings.receive_timeout) - .withTCPKeepAliveTimeout(settings.tcp_keep_alive_timeout) - .withHandshakeTimeout(settings.handshake_timeout_ms) - .withHedgedConnectionTimeout(settings.hedged_connection_timeout_ms) - .withReceiveDataTimeout(settings.receive_data_timeout_ms); + .withConnectionTimeout(settings[Setting::connect_timeout]) + .withSendTimeout(settings[Setting::send_timeout]) + .withReceiveTimeout(settings[Setting::receive_timeout]) + .withTCPKeepAliveTimeout(settings[Setting::tcp_keep_alive_timeout]) + .withHandshakeTimeout(settings[Setting::handshake_timeout_ms]) + .withHedgedConnectionTimeout(settings[Setting::hedged_connection_timeout_ms]) + .withReceiveDataTimeout(settings[Setting::receive_data_timeout_ms]); } /// Timeouts for the case when we will try many addresses in a loop. ConnectionTimeouts ConnectionTimeouts::getTCPTimeoutsWithFailover(const Settings & settings) { return getTCPTimeoutsWithoutFailover(settings) - .withUnsecureConnectionTimeout(settings.connect_timeout_with_failover_ms) - .withSecureConnectionTimeout(settings.connect_timeout_with_failover_secure_ms); + .withUnsecureConnectionTimeout(settings[Setting::connect_timeout_with_failover_ms]) + .withSecureConnectionTimeout(settings[Setting::connect_timeout_with_failover_secure_ms]); } ConnectionTimeouts ConnectionTimeouts::getHTTPTimeouts(const Settings & settings, Poco::Timespan http_keep_alive_timeout) { return ConnectionTimeouts() - .withConnectionTimeout(settings.http_connection_timeout) - .withSendTimeout(settings.http_send_timeout) - .withReceiveTimeout(settings.http_receive_timeout) + .withConnectionTimeout(settings[Setting::http_connection_timeout]) + .withSendTimeout(settings[Setting::http_send_timeout]) + .withReceiveTimeout(settings[Setting::http_receive_timeout]) .withHTTPKeepAliveTimeout(http_keep_alive_timeout) - .withTCPKeepAliveTimeout(settings.tcp_keep_alive_timeout) - .withHandshakeTimeout(settings.handshake_timeout_ms); + .withTCPKeepAliveTimeout(settings[Setting::tcp_keep_alive_timeout]) + .withHandshakeTimeout(settings[Setting::handshake_timeout_ms]); } ConnectionTimeouts ConnectionTimeouts::getFetchPartHTTPTimeouts(const ServerSettings & server_settings, const Settings & user_settings) diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index d4c41a3f2cd..9a0eccd8783 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -51,6 +51,10 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool s3_use_adaptive_timeouts; +} namespace ErrorCodes { @@ -995,7 +999,7 @@ PocoHTTPClientConfiguration ClientFactory::createClientConfiguration( // NOLINT s3_retry_attempts, enable_s3_requests_logging, for_disk_s3, - context->getGlobalContext()->getSettingsRef().s3_use_adaptive_timeouts, + context->getGlobalContext()->getSettingsRef()[Setting::s3_use_adaptive_timeouts], get_request_throttler, put_request_throttler, error_report); diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 73bbba055d0..774cb42fab4 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -9,6 +9,7 @@ #include #include +#include namespace DB diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 59040bf1fea..af5e0339a9f 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -7,7 +7,9 @@ #include #include #include + #include +#include #include "config.h" @@ -57,6 +59,13 @@ namespace DB::ErrorCodes namespace DB { +namespace Setting +{ + extern const SettingsUInt64 s3_max_get_burst; + extern const SettingsUInt64 s3_max_get_rps; + extern const SettingsUInt64 s3_max_put_burst; + extern const SettingsUInt64 s3_max_put_rps; +} namespace ErrorCodes { @@ -294,20 +303,18 @@ void RequestSettings::finishInit(const DB::Settings & settings, bool validate_se /// to avoid losing token bucket state on every config reload, /// which could lead to exceeding limit for short time. /// But it is good enough unless very high `burst` values are used. - if (UInt64 max_get_rps = isChanged("max_get_rps") ? get("max_get_rps").safeGet() : settings.s3_max_get_rps) + if (UInt64 max_get_rps = isChanged("max_get_rps") ? get("max_get_rps").safeGet() : settings[Setting::s3_max_get_rps]) { - size_t default_max_get_burst = settings.s3_max_get_burst - ? settings.s3_max_get_burst - : (Throttler::default_burst_seconds * max_get_rps); + size_t default_max_get_burst + = settings[Setting::s3_max_get_burst] ? settings[Setting::s3_max_get_burst] : (Throttler::default_burst_seconds * max_get_rps); size_t max_get_burst = isChanged("max_get_burts") ? get("max_get_burst").safeGet() : default_max_get_burst; get_request_throttler = std::make_shared(max_get_rps, max_get_burst); } - if (UInt64 max_put_rps = isChanged("max_put_rps") ? get("max_put_rps").safeGet() : settings.s3_max_put_rps) + if (UInt64 max_put_rps = isChanged("max_put_rps") ? get("max_put_rps").safeGet() : settings[Setting::s3_max_put_rps]) { - size_t default_max_put_burst = settings.s3_max_put_burst - ? settings.s3_max_put_burst - : (Throttler::default_burst_seconds * max_put_rps); + size_t default_max_put_burst + = settings[Setting::s3_max_put_burst] ? settings[Setting::s3_max_put_burst] : (Throttler::default_burst_seconds * max_put_rps); size_t max_put_burst = isChanged("max_put_burts") ? get("max_put_burst").safeGet() : default_max_put_burst; put_request_throttler = std::make_shared(max_put_rps, max_put_burst); } diff --git a/src/IO/S3Settings.cpp b/src/IO/S3Settings.cpp index f99e8c0105b..3c960258b4c 100644 --- a/src/IO/S3Settings.cpp +++ b/src/IO/S3Settings.cpp @@ -9,6 +9,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool s3_validate_request_settings; +} void S3Settings::loadFromConfig( const Poco::Util::AbstractConfiguration & config, @@ -50,7 +54,7 @@ void S3SettingsByEndpoint::loadFromConfig( auth_settings.updateIfChanged(S3::AuthSettings(config, settings, key_path)); auto request_settings{default_request_settings}; - request_settings.updateIfChanged(S3::RequestSettings(config, settings, key_path, "", settings.s3_validate_request_settings)); + request_settings.updateIfChanged(S3::RequestSettings(config, settings, key_path, "", settings[Setting::s3_validate_request_settings])); s3_settings.emplace( config.getString(endpoint_path), diff --git a/src/IO/examples/CMakeLists.txt b/src/IO/examples/CMakeLists.txt index bfd171a3d00..48fb9ce858b 100644 --- a/src/IO/examples/CMakeLists.txt +++ b/src/IO/examples/CMakeLists.txt @@ -38,7 +38,7 @@ clickhouse_add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (io_operators io_operators.cpp) -target_link_libraries (io_operators PRIVATE clickhouse_common_io clickhouse_common_config) +target_link_libraries (io_operators PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (write_int write_int.cpp) target_link_libraries (write_int PRIVATE clickhouse_common_io clickhouse_common_config) @@ -56,7 +56,7 @@ clickhouse_add_executable (limit_read_buffer2 limit_read_buffer2.cpp) target_link_libraries (limit_read_buffer2 PRIVATE clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (parse_date_time_best_effort parse_date_time_best_effort.cpp) -target_link_libraries (parse_date_time_best_effort PRIVATE clickhouse_common_io clickhouse_common_config) +target_link_libraries (parse_date_time_best_effort PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config) clickhouse_add_executable (zlib_ng_bug zlib_ng_bug.cpp) target_link_libraries (zlib_ng_bug PRIVATE ch_contrib::zlib clickhouse_common_io clickhouse_common_config) @@ -75,7 +75,7 @@ target_link_libraries (hadoop_snappy_read_buffer PRIVATE clickhouse_common_io cl if (TARGET ch_contrib::hdfs) clickhouse_add_executable (read_buffer_from_hdfs read_buffer_from_hdfs.cpp) - target_link_libraries (read_buffer_from_hdfs PRIVATE dbms ch_contrib::hdfs) + target_link_libraries (read_buffer_from_hdfs PRIVATE dbms clickhouse_functions ch_contrib::hdfs) endif () diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index b53a8b58023..a796f87f5bc 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -40,6 +40,17 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool s3_check_objects_after_upload; + extern const SettingsUInt64 s3_max_inflight_parts_for_one_file; + extern const SettingsUInt64 s3_max_single_part_upload_size; + extern const SettingsUInt64 s3_max_upload_part_size; + extern const SettingsUInt64 s3_min_upload_part_size; + extern const SettingsUInt64 s3_strict_upload_part_size; + extern const SettingsUInt64 s3_upload_part_size_multiply_factor; + extern const SettingsUInt64 s3_upload_part_size_multiply_parts_count_threshold; +} namespace ErrorCodes { @@ -641,7 +652,7 @@ protected: { /// Do not block the main thread awaiting the others task. /// This test use the only one thread at all - getSettings().s3_max_inflight_parts_for_one_file = 0; + getSettings()[Setting::s3_max_inflight_parts_for_one_file] = 0; async_policy = std::make_unique(); } else @@ -662,7 +673,7 @@ INSTANTIATE_TEST_SUITE_P(WBS3 TEST_P(SyncAsync, ExceptionOnHead) { setInjectionModel(std::make_shared()); - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; EXPECT_THROW({ try { @@ -742,8 +753,8 @@ TEST_P(SyncAsync, ExceptionOnPut) { TEST_P(SyncAsync, ExceptionOnCreateMPU) { setInjectionModel(std::make_shared()); - getSettings().s3_max_single_part_upload_size = 0; // no single part - getSettings().s3_min_upload_part_size = 1; // small parts ara ok + getSettings()[Setting::s3_max_single_part_upload_size] = 0; // no single part + getSettings()[Setting::s3_min_upload_part_size] = 1; // small parts ara ok EXPECT_THROW({ try { @@ -802,8 +813,8 @@ TEST_P(SyncAsync, ExceptionOnCreateMPU) { TEST_P(SyncAsync, ExceptionOnCompleteMPU) { setInjectionModel(std::make_shared()); - getSettings().s3_max_single_part_upload_size = 0; // no single part - getSettings().s3_min_upload_part_size = 1; // small parts ara ok + getSettings()[Setting::s3_max_single_part_upload_size] = 0; // no single part + getSettings()[Setting::s3_min_upload_part_size] = 1; // small parts ara ok EXPECT_THROW({ try { @@ -825,8 +836,8 @@ TEST_P(SyncAsync, ExceptionOnCompleteMPU) { TEST_P(SyncAsync, ExceptionOnUploadPart) { setInjectionModel(std::make_shared()); - getSettings().s3_max_single_part_upload_size = 0; // no single part - getSettings().s3_min_upload_part_size = 1; // small parts ara ok + getSettings()[Setting::s3_max_single_part_upload_size] = 0; // no single part + getSettings()[Setting::s3_min_upload_part_size] = 1; // small parts ara ok MockS3::EventCounts counters = {.multiUploadCreate = 1, .multiUploadAbort = 1}; @@ -942,14 +953,14 @@ TEST_F(WBS3Test, PrefinalizeCalledMultipleTimes) { } TEST_P(SyncAsync, EmptyFile) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; MockS3::EventCounts counters = {.headObject = 2, .putObject = 1}; runSimpleScenario(counters, 0); } TEST_P(SyncAsync, ManualNextCalls) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; { MockS3::EventCounts counters = {.headObject = 2, .putObject = 1}; @@ -1008,101 +1019,100 @@ TEST_P(SyncAsync, ManualNextCalls) { } TEST_P(SyncAsync, SmallFileIsOnePutRequest) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; { - getSettings().s3_max_single_part_upload_size = 1000; - getSettings().s3_min_upload_part_size = 10; + getSettings()[Setting::s3_max_single_part_upload_size] = 1000; + getSettings()[Setting::s3_min_upload_part_size] = 10; MockS3::EventCounts counters = {.headObject = 2, .putObject = 1}; runSimpleScenario(counters, 1); - runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size-1); - runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size); - runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size/2); + runSimpleScenario(counters, getSettings()[Setting::s3_max_single_part_upload_size] - 1); + runSimpleScenario(counters, getSettings()[Setting::s3_max_single_part_upload_size]); + runSimpleScenario(counters, getSettings()[Setting::s3_max_single_part_upload_size] / 2); } { - - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_min_upload_part_size = 1000; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_min_upload_part_size] = 1000; MockS3::EventCounts counters = {.headObject = 2, .putObject = 1}; runSimpleScenario(counters, 1); - runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size-1); - runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size); - runSimpleScenario(counters, getSettings().s3_max_single_part_upload_size/2); + runSimpleScenario(counters, getSettings()[Setting::s3_max_single_part_upload_size] - 1); + runSimpleScenario(counters, getSettings()[Setting::s3_max_single_part_upload_size]); + runSimpleScenario(counters, getSettings()[Setting::s3_max_single_part_upload_size] / 2); } } TEST_P(SyncAsync, LittleBiggerFileIsMultiPartUpload) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; { - getSettings().s3_max_single_part_upload_size = 1000; - getSettings().s3_min_upload_part_size = 10; + getSettings()[Setting::s3_max_single_part_upload_size] = 1000; + getSettings()[Setting::s3_min_upload_part_size] = 10; MockS3::EventCounts counters = {.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 2}; - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 1); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + 1); counters.uploadParts = 101; - runSimpleScenario(counters, 2*settings.s3_max_single_part_upload_size); + runSimpleScenario(counters, 2 * settings[Setting::s3_max_single_part_upload_size]); } { - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_min_upload_part_size = 1000; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_min_upload_part_size] = 1000; MockS3::EventCounts counters = {.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 1}; - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 1); - runSimpleScenario(counters, 2*settings.s3_max_single_part_upload_size); - runSimpleScenario(counters, settings.s3_min_upload_part_size-1); - runSimpleScenario(counters, settings.s3_min_upload_part_size); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + 1); + runSimpleScenario(counters, 2 * settings[Setting::s3_max_single_part_upload_size]); + runSimpleScenario(counters, settings[Setting::s3_min_upload_part_size] - 1); + runSimpleScenario(counters, settings[Setting::s3_min_upload_part_size]); } } TEST_P(SyncAsync, BiggerFileIsMultiPartUpload) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; { - getSettings().s3_max_single_part_upload_size = 1000; - getSettings().s3_min_upload_part_size = 10; + getSettings()[Setting::s3_max_single_part_upload_size] = 1000; + getSettings()[Setting::s3_min_upload_part_size] = 10; auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 2}; - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + settings[Setting::s3_min_upload_part_size]); counters.uploadParts = 3; - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size + 1); - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 2*settings.s3_min_upload_part_size - 1); - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + 2*settings.s3_min_upload_part_size); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + settings[Setting::s3_min_upload_part_size] + 1); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + 2 * settings[Setting::s3_min_upload_part_size] - 1); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + 2 * settings[Setting::s3_min_upload_part_size]); } { // but not in that case, when s3_min_upload_part_size > s3_max_single_part_upload_size - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_min_upload_part_size = 1000; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_min_upload_part_size] = 1000; auto counters = MockS3::EventCounts{.headObject = 2, .multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 2}; - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size); - runSimpleScenario(counters, settings.s3_max_single_part_upload_size + settings.s3_min_upload_part_size + 1); - runSimpleScenario(counters, 2*settings.s3_min_upload_part_size-1); - runSimpleScenario(counters, 2*settings.s3_min_upload_part_size); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + settings[Setting::s3_min_upload_part_size]); + runSimpleScenario(counters, settings[Setting::s3_max_single_part_upload_size] + settings[Setting::s3_min_upload_part_size] + 1); + runSimpleScenario(counters, 2 * settings[Setting::s3_min_upload_part_size] - 1); + runSimpleScenario(counters, 2 * settings[Setting::s3_min_upload_part_size]); counters.uploadParts = 3; - runSimpleScenario(counters, 2*settings.s3_min_upload_part_size+1); + runSimpleScenario(counters, 2 * settings[Setting::s3_min_upload_part_size] + 1); } } TEST_P(SyncAsync, IncreaseUploadBuffer) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; { - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_min_upload_part_size = 10; - getSettings().s3_upload_part_size_multiply_parts_count_threshold = 1; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_min_upload_part_size] = 10; + getSettings()[Setting::s3_upload_part_size_multiply_parts_count_threshold] = 1; // parts: 10 20 40 80 160 // size: 10 30 70 150 310 @@ -1114,10 +1124,10 @@ TEST_P(SyncAsync, IncreaseUploadBuffer) { } { - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_min_upload_part_size = 10; - getSettings().s3_upload_part_size_multiply_parts_count_threshold = 2; - getSettings().s3_upload_part_size_multiply_factor = 3; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_min_upload_part_size] = 10; + getSettings()[Setting::s3_upload_part_size_multiply_parts_count_threshold] = 2; + getSettings()[Setting::s3_upload_part_size_multiply_factor] = 3; // parts: 10 10 30 30 90 // size: 10 20 50 80 170 @@ -1130,13 +1140,13 @@ TEST_P(SyncAsync, IncreaseUploadBuffer) { } TEST_P(SyncAsync, IncreaseLimited) { - getSettings().s3_check_objects_after_upload = true; + getSettings()[Setting::s3_check_objects_after_upload] = true; { - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_min_upload_part_size = 10; - getSettings().s3_upload_part_size_multiply_parts_count_threshold = 1; - getSettings().s3_max_upload_part_size = 45; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_min_upload_part_size] = 10; + getSettings()[Setting::s3_upload_part_size_multiply_parts_count_threshold] = 1; + getSettings()[Setting::s3_max_upload_part_size] = 45; // parts: 10 20 40 45 45 45 // size: 10 30 70 115 160 205 @@ -1149,11 +1159,11 @@ TEST_P(SyncAsync, IncreaseLimited) { } TEST_P(SyncAsync, StrictUploadPartSize) { - getSettings().s3_check_objects_after_upload = false; + getSettings()[Setting::s3_check_objects_after_upload] = false; { - getSettings().s3_max_single_part_upload_size = 10; - getSettings().s3_strict_upload_part_size = 11; + getSettings()[Setting::s3_max_single_part_upload_size] = 10; + getSettings()[Setting::s3_strict_upload_part_size] = 11; { auto counters = MockS3::EventCounts{.multiUploadCreate = 1, .multiUploadComplete = 1, .uploadParts = 6}; diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 368eb8174f0..980222849e2 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1,10 +1,9 @@ #include +#include +#include #include #include -#include -#include -#include #include #include @@ -42,18 +41,17 @@ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -63,6 +61,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool force_grouping_standard_compatibility; + extern const SettingsUInt64 max_ast_elements; + extern const SettingsBool transform_null_in; + extern const SettingsBool use_variant_as_common_type; +} namespace ErrorCodes { @@ -197,7 +204,7 @@ static Block createBlockFromAST(const ASTPtr & node, const DataTypes & types, Co DataTypePtr tuple_type; Row tuple_values; const auto & list = node->as(); - bool transform_null_in = context->getSettingsRef().transform_null_in; + bool transform_null_in = context->getSettingsRef()[Setting::transform_null_in]; for (const auto & elem : list.children) { if (num_columns == 1) @@ -316,7 +323,7 @@ Block createBlockForSet( }; Block block; - bool tranform_null_in = context->getSettingsRef().transform_null_in; + bool tranform_null_in = context->getSettingsRef()[Setting::transform_null_in]; /// 1 in 1; (1, 2) in (1, 2); identity(tuple(tuple(tuple(1)))) in tuple(tuple(tuple(1))); etc. if (left_type_depth == right_type_depth) @@ -426,7 +433,7 @@ FutureSetPtr makeExplicitSet( if (left_tuple_type && left_tuple_type->getElements().size() != 1) set_element_types = left_tuple_type->getElements(); - auto set_element_keys = Set::getElementTypes(set_element_types, context->getSettingsRef().transform_null_in); + auto set_element_keys = Set::getElementTypes(set_element_types, context->getSettingsRef()[Setting::transform_null_in]); auto set_key = right_arg->getTreeHash(/*ignore_aliases=*/ true); if (auto set = prepared_sets.findTuple(set_key, set_element_keys)) @@ -782,7 +789,7 @@ ASTs ActionsMatcher::doUntuple(const ASTFunction * function, ActionsMatcher::Dat auto tuple_ast = function->arguments->children[0]; /// This transformation can lead to exponential growth of AST size, let's check it. - tuple_ast->checkSize(data.getContext()->getSettingsRef().max_ast_elements); + tuple_ast->checkSize(data.getContext()->getSettingsRef()[Setting::max_ast_elements]); if (tid != 0) tuple_ast = tuple_ast->clone(); @@ -926,20 +933,20 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { case GroupByKind::GROUPING_SETS: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), keys_info.grouping_set_keys, data.getContext()->getSettingsRef()[Setting::force_grouping_standard_compatibility])), { "__grouping_set" }, column_name); break; } case GroupByKind::ROLLUP: - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef()[Setting::force_grouping_standard_compatibility])), { "__grouping_set" }, column_name); break; case GroupByKind::CUBE: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), { "__grouping_set" }, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), aggregation_keys_number, data.getContext()->getSettingsRef()[Setting::force_grouping_standard_compatibility])), { "__grouping_set" }, column_name); break; } case GroupByKind::ORDINARY: { - data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.getContext()->getSettingsRef().force_grouping_standard_compatibility)), {}, column_name); + data.addFunction(std::make_shared(std::make_shared(std::move(arguments_indexes), data.getContext()->getSettingsRef()[Setting::force_grouping_standard_compatibility])), {}, column_name); break; } default: @@ -1338,7 +1345,9 @@ void ActionsMatcher::visit(const ASTLiteral & literal, const ASTPtr & /* ast */, DataTypePtr type; if (literal.custom_type) type = literal.custom_type; - else if (data.getContext()->getSettingsRef().allow_experimental_variant_type && data.getContext()->getSettingsRef().use_variant_as_common_type) + else if ( + data.getContext()->getSettingsRef()[Setting::allow_experimental_variant_type] + && data.getContext()->getSettingsRef()[Setting::use_variant_as_common_type]) type = applyVisitor(FieldToDataType(), literal.value); else type = applyVisitor(FieldToDataType(), literal.value); @@ -1411,7 +1420,7 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool return {}; PreparedSets::Hash set_key; - if (data.getContext()->getSettingsRef().allow_experimental_analyzer && !identifier) + if (data.getContext()->getSettingsRef()[Setting::allow_experimental_analyzer] && !identifier) { /// Here we can be only from mutation interpreter. Normal selects with analyzed use other interpreter. /// This is a hacky way to allow reusing cache for prepared sets. diff --git a/src/Interpreters/ArrayJoinAction.cpp b/src/Interpreters/ArrayJoinAction.cpp index f49b3af9c58..bdec5d1025e 100644 --- a/src/Interpreters/ArrayJoinAction.cpp +++ b/src/Interpreters/ArrayJoinAction.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 25cd1d0bfa2..736cd157c1e 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -56,6 +56,33 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsBool allow_experimental_query_deduplication; + extern const SettingsDouble async_insert_busy_timeout_decrease_rate; + extern const SettingsDouble async_insert_busy_timeout_increase_rate; + extern const SettingsMilliseconds async_insert_busy_timeout_min_ms; + extern const SettingsMilliseconds async_insert_busy_timeout_max_ms; + extern const SettingsBool async_insert_deduplicate; + extern const SettingsUInt64 async_insert_max_data_size; + extern const SettingsUInt64 async_insert_max_query_number; + extern const SettingsMilliseconds async_insert_poll_timeout_ms; + extern const SettingsBool async_insert_use_adaptive_busy_timeout; + extern const SettingsBool empty_result_for_aggregation_by_empty_set; + extern const SettingsBool insert_allow_materialized_columns; + extern const SettingsString insert_deduplication_token; + extern const SettingsBool input_format_defaults_for_omitted_fields; + extern const SettingsUInt64 log_queries_cut_to_length; + extern const SettingsUInt64 max_columns_to_read; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsBool optimize_trivial_count_query; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsString parallel_replicas_custom_key; + extern const SettingsParallelReplicasCustomKeyFilterType parallel_replicas_custom_key_filter_type; + extern const SettingsUInt64 parallel_replicas_custom_key_range_lower; + extern const SettingsUInt64 parallel_replica_offset; +} namespace ErrorCodes { @@ -100,14 +127,15 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( siphash.update(current_role); } - for (const auto & setting : settings.allChanged()) + auto changes = settings.changes(); + for (const auto & change : changes) { - if (settings_to_skip.contains(setting.getName())) + if (settings_to_skip.contains(change.name)) continue; - setting_changes.emplace_back(setting.getName(), setting.getValue()); - siphash.update(setting.getName()); - applyVisitor(FieldVisitorHash(siphash), setting.getValue()); + setting_changes.emplace_back(change.name, change.value); + siphash.update(change.name); + applyVisitor(FieldVisitorHash(siphash), change.value); } hash = siphash.get128(); @@ -215,7 +243,7 @@ AsynchronousInsertQueue::AsynchronousInsertQueue(ContextPtr context_, size_t poo for (size_t i = 0; i < pool_size; ++i) queue_shards[i].busy_timeout_ms - = std::min(Milliseconds(settings.async_insert_busy_timeout_min_ms), Milliseconds(settings.async_insert_busy_timeout_max_ms)); + = std::min(Milliseconds(settings[Setting::async_insert_busy_timeout_min_ms]), Milliseconds(settings[Setting::async_insert_busy_timeout_max_ms])); for (size_t i = 0; i < pool_size; ++i) dump_by_first_update_threads.emplace_back([this, i] { processBatchDeadlines(i); }); @@ -306,7 +334,7 @@ void AsynchronousInsertQueue::preprocessInsertQuery(const ASTPtr & query, const InterpreterInsertQuery interpreter( query, query_context, - query_context->getSettingsRef().insert_allow_materialized_columns, + query_context->getSettingsRef()[Setting::insert_allow_materialized_columns], /* no_squash */ false, /* no_destination */ false, /* async_insert */ false); @@ -353,8 +381,10 @@ AsynchronousInsertQueue::pushQueryWithInlinedData(ASTPtr query, ContextPtr query auto read_buf = getReadBufferFromASTInsertQuery(query); LimitReadBuffer limit_buf( - *read_buf, query_context->getSettingsRef().async_insert_max_data_size, - /*throw_exception=*/ false, /*exact_limit=*/ {}); + *read_buf, + query_context->getSettingsRef()[Setting::async_insert_max_data_size], + /*throw_exception=*/false, + /*exact_limit=*/{}); WriteBufferFromString write_buf(bytes); copyData(limit_buf, write_buf); @@ -398,8 +428,10 @@ AsynchronousInsertQueue::pushDataChunk(ASTPtr query, DataChunk chunk, ContextPtr auto data_kind = chunk.getDataKind(); auto entry = std::make_shared( - std::move(chunk), query_context->getCurrentQueryId(), - settings.insert_deduplication_token, insert_query.format, + std::move(chunk), + query_context->getCurrentQueryId(), + settings[Setting::insert_deduplication_token], + insert_query.format, CurrentThread::getUserMemoryTracker()); /// If data is parsed on client we don't care of format which is written @@ -452,15 +484,16 @@ AsynchronousInsertQueue::pushDataChunk(ASTPtr query, DataChunk chunk, ContextPtr LOG_TRACE(log, "Have {} pending inserts with total {} bytes of data for query '{}'", data->entries.size(), data->size_in_bytes, key.query_str); - bool has_enough_bytes = data->size_in_bytes >= key.settings.async_insert_max_data_size; - bool has_enough_queries = data->entries.size() >= key.settings.async_insert_max_query_number && key.settings.async_insert_deduplicate; + bool has_enough_bytes = data->size_in_bytes >= key.settings[Setting::async_insert_max_data_size]; + bool has_enough_queries + = data->entries.size() >= key.settings[Setting::async_insert_max_query_number] && key.settings[Setting::async_insert_deduplicate]; auto max_busy_timeout_exceeded = [&shard, &settings, &now, &flush_time_points]() -> bool { - if (!settings.async_insert_use_adaptive_busy_timeout || !shard.last_insert_time || !flush_time_points.first) + if (!settings[Setting::async_insert_use_adaptive_busy_timeout] || !shard.last_insert_time || !flush_time_points.first) return false; - auto max_ms = Milliseconds(settings.async_insert_busy_timeout_max_ms); + auto max_ms = Milliseconds(settings[Setting::async_insert_busy_timeout_max_ms]); return *shard.last_insert_time + max_ms < now && *flush_time_points.first + max_ms < *flush_time_points.second; }; @@ -516,11 +549,11 @@ AsynchronousInsertQueue::Milliseconds AsynchronousInsertQueue::getBusyWaitTimeou const QueueShardFlushTimeHistory::TimePoints & flush_time_points, std::chrono::steady_clock::time_point now) const { - if (!settings.async_insert_use_adaptive_busy_timeout) - return settings.async_insert_busy_timeout_max_ms; + if (!settings[Setting::async_insert_use_adaptive_busy_timeout]) + return settings[Setting::async_insert_busy_timeout_max_ms]; - const auto max_ms = Milliseconds(settings.async_insert_busy_timeout_max_ms); - const auto min_ms = std::min(std::max(Milliseconds(settings.async_insert_busy_timeout_min_ms), Milliseconds(1)), max_ms); + const auto max_ms = Milliseconds(settings[Setting::async_insert_busy_timeout_max_ms]); + const auto min_ms = std::min(std::max(Milliseconds(settings[Setting::async_insert_busy_timeout_min_ms]), Milliseconds(1)), max_ms); auto normalize = [&min_ms, &max_ms](const auto & t_ms) { return std::min(std::max(t_ms, min_ms), max_ms); }; @@ -529,8 +562,8 @@ AsynchronousInsertQueue::Milliseconds AsynchronousInsertQueue::getBusyWaitTimeou const auto & last_insert_time = *shard.last_insert_time; const auto & [t1, t2] = std::tie(*flush_time_points.first, *flush_time_points.second); - const double increase_rate = settings.async_insert_busy_timeout_increase_rate; - const double decrease_rate = settings.async_insert_busy_timeout_decrease_rate; + const double increase_rate = settings[Setting::async_insert_busy_timeout_increase_rate]; + const double decrease_rate = settings[Setting::async_insert_busy_timeout_decrease_rate]; const auto decreased_timeout_ms = std::min( std::chrono::duration_cast(shard.busy_timeout_ms / (1.0 + decrease_rate)), shard.busy_timeout_ms - Milliseconds(1)); @@ -556,16 +589,16 @@ AsynchronousInsertQueue::Milliseconds AsynchronousInsertQueue::getBusyWaitTimeou void AsynchronousInsertQueue::validateSettings(const Settings & settings, LoggerPtr log) { - const auto max_ms = std::chrono::milliseconds(settings.async_insert_busy_timeout_max_ms); + const auto max_ms = std::chrono::milliseconds(settings[Setting::async_insert_busy_timeout_max_ms]); if (max_ms == std::chrono::milliseconds::zero()) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting 'async_insert_busy_timeout_max_ms' can't be zero"); - if (!settings.async_insert_use_adaptive_busy_timeout) + if (!settings[Setting::async_insert_use_adaptive_busy_timeout]) return; /// Adaptive timeout settings. - const auto min_ms = std::chrono::milliseconds(settings.async_insert_busy_timeout_min_ms); + const auto min_ms = std::chrono::milliseconds(settings[Setting::async_insert_busy_timeout_min_ms]); if (min_ms > max_ms && log) LOG_WARNING( @@ -575,10 +608,10 @@ void AsynchronousInsertQueue::validateSettings(const Settings & settings, Logger min_ms.count(), max_ms.count()); - if (settings.async_insert_busy_timeout_increase_rate <= 0) + if (settings[Setting::async_insert_busy_timeout_increase_rate] <= 0) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting 'async_insert_busy_timeout_increase_rate' must be greater than zero"); - if (settings.async_insert_busy_timeout_decrease_rate <= 0) + if (settings[Setting::async_insert_busy_timeout_decrease_rate] <= 0) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting 'async_insert_busy_timeout_decrease_rate' must be greater than zero"); } @@ -639,7 +672,7 @@ void AsynchronousInsertQueue::processBatchDeadlines(size_t shard_num) std::unique_lock lock(shard.mutex); const auto rel_time - = std::min(shard.busy_timeout_ms, Milliseconds(getContext()->getSettingsRef().async_insert_poll_timeout_ms)); + = std::min(shard.busy_timeout_ms, Milliseconds(getContext()->getSettingsRef()[Setting::async_insert_poll_timeout_ms])); shard.are_tasks_available.wait_for( lock, rel_time, @@ -786,7 +819,7 @@ try DB::CurrentThread::QueryScope query_scope_holder(insert_context); - auto query_for_logging = serializeQuery(*key.query, insert_context->getSettingsRef().log_queries_cut_to_length); + auto query_for_logging = serializeQuery(*key.query, insert_context->getSettingsRef()[Setting::log_queries_cut_to_length]); /// We add it to the process list so /// a) it appears in system.processes @@ -851,7 +884,7 @@ try auto query = key.query->clone(); assert_cast(*query).format = format; - it->second = serializeQuery(*query, insert_context->getSettingsRef().log_queries_cut_to_length); + it->second = serializeQuery(*query, insert_context->getSettingsRef()[Setting::log_queries_cut_to_length]); return it->second; }; @@ -878,12 +911,7 @@ try try { interpreter = std::make_unique( - key.query, - insert_context, - key.settings.insert_allow_materialized_columns, - false, - false, - true); + key.query, insert_context, key.settings[Setting::insert_allow_materialized_columns], false, false, true); pipeline = interpreter->execute().pipeline; chassert(pipeline.pushing()); @@ -1014,7 +1042,7 @@ Chunk AsynchronousInsertQueue::processEntriesWithParsing( auto format = getInputFormatFromASTInsertQuery(key.query, false, header, insert_context, nullptr); std::shared_ptr adding_defaults_transform; - if (insert_context->getSettingsRef().input_format_defaults_for_omitted_fields && insert_query.table_id) + if (insert_context->getSettingsRef()[Setting::input_format_defaults_for_omitted_fields] && insert_query.table_id) { StoragePtr storage = DatabaseCatalog::instance().getTable(insert_query.table_id, insert_context); auto metadata_snapshot = storage->getInMemoryMetadataPtr(); diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index ecb7757d406..eb4e1ed471c 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -3,11 +3,12 @@ #include #include #include +#include #include #include #include +#include #include -#include #include #include diff --git a/src/Interpreters/Cache/QueryCache.cpp b/src/Interpreters/Cache/QueryCache.cpp index 4312b35e18c..c766c5209fc 100644 --- a/src/Interpreters/Cache/QueryCache.cpp +++ b/src/Interpreters/Cache/QueryCache.cpp @@ -30,6 +30,10 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsString query_cache_tag; +} namespace { @@ -197,14 +201,13 @@ IAST::Hash calculateAstHash(ASTPtr ast, const String & current_database, const S /// Finally, hash the (changed) settings as they might affect the query result (e.g. think of settings `additional_table_filters` and `limit`). /// Note: allChanged() returns the settings in random order. Also, update()-s of the composite hash must be done in deterministic order. /// Therefore, collect and sort the settings first, then hash them. - Settings::Range changed_settings = settings.allChanged(); + auto changed_settings = settings.changes(); std::vector> changed_settings_sorted; /// (name, value) - for (const auto & setting : changed_settings) + for (const auto & change : changed_settings) { - const String & name = setting.getName(); - const String & value = setting.getValueString(); + const String & name = change.name; if (!isQueryCacheRelatedSetting(name)) /// see removeQueryCacheSettings() why this is a good idea - changed_settings_sorted.push_back({name, value}); + changed_settings_sorted.push_back({name, Settings::valueToStringUtil(change.name, change.value)}); } std::sort(changed_settings_sorted.begin(), changed_settings_sorted.end(), [](auto & lhs, auto & rhs) { return lhs.first < rhs.first; }); for (const auto & setting : changed_settings_sorted) @@ -230,7 +233,8 @@ QueryCache::Key::Key( const String & current_database, const Settings & settings, Block header_, - std::optional user_id_, const std::vector & current_user_roles_, + std::optional user_id_, + const std::vector & current_user_roles_, bool is_shared_, std::chrono::time_point expires_at_, bool is_compressed_) @@ -242,7 +246,7 @@ QueryCache::Key::Key( , expires_at(expires_at_) , is_compressed(is_compressed_) , query_string(queryStringFromAST(ast_)) - , tag(settings.query_cache_tag) + , tag(settings[Setting::query_cache_tag]) { } diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index e43bbacdbc5..5bdf32de68f 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -16,6 +16,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds; +} namespace ErrorCodes { @@ -29,9 +33,10 @@ namespace { auto query_context = CurrentThread::getQueryContext(); if (query_context) - return query_context->getSettingsRef().temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds; + return query_context->getSettingsRef()[Setting::temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds]; else - return Context::getGlobalContextInstance()->getSettingsRef().temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds; + return Context::getGlobalContextInstance() + ->getSettingsRef()[Setting::temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds]; } } diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 7b7bedc850d..d10ca6d3961 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -1,26 +1,33 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include -#include +#include +#include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 distributed_connections_pool_size; + extern const SettingsUInt64 distributed_replica_error_cap; + extern const SettingsSeconds distributed_replica_error_half_life; + extern const SettingsLoadBalancing load_balancing; + extern const SettingsBool prefer_localhost_replica; +} namespace ErrorCodes { @@ -446,17 +453,23 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, info.local_addresses.push_back(address); auto pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), - address.host_name, address.port, - address.default_database, address.user, address.password, - address.proto_send_chunked, address.proto_recv_chunked, + static_cast(settings[Setting::distributed_connections_pool_size]), + address.host_name, + address.port, + address.default_database, + address.user, + address.password, + address.proto_send_chunked, + address.proto_recv_chunked, address.quota_key, - address.cluster, address.cluster_secret, - "server", address.compression, - address.secure, address.priority); + address.cluster, + address.cluster_secret, + "server", + address.compression, + address.secure, + address.priority); - info.pool = std::make_shared( - ConnectionPoolPtrs{pool}, settings.load_balancing); + info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings[Setting::load_balancing]); info.per_replica_pools = {std::move(pool)}; if (weight) @@ -608,7 +621,7 @@ void Cluster::addShard( for (const auto & replica : addresses) { auto replica_pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), + static_cast(settings[Setting::distributed_connections_pool_size]), replica.host_name, replica.port, replica.default_database, @@ -630,9 +643,9 @@ void Cluster::addShard( } ConnectionPoolWithFailoverPtr shard_pool = std::make_shared( all_replicas_pools, - settings.load_balancing, - settings.distributed_replica_error_half_life.totalSeconds(), - settings.distributed_replica_error_cap); + settings[Setting::load_balancing], + settings[Setting::distributed_replica_error_half_life].totalSeconds(), + settings[Setting::distributed_replica_error_cap]); if (weight) slot_to_shard.insert(std::end(slot_to_shard), weight, shards_info.size()); @@ -712,7 +725,7 @@ void shuffleReplicas(std::vector & replicas, const Settings & { pcg64_fast gen{randomSeed()}; - if (settings.prefer_localhost_replica) + if (settings[Setting::prefer_localhost_replica]) { // force for local replica to always be included auto first_non_local_replica = std::partition(replicas.begin(), replicas.end(), [](const auto & replica) { return replica.is_local; }); @@ -765,7 +778,7 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti info.local_addresses.push_back(address); auto pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), + static_cast(settings[Setting::distributed_connections_pool_size]), address.host_name, address.port, address.default_database, @@ -781,7 +794,7 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti address.secure, address.priority); - info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings.load_balancing); + info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings[Setting::load_balancing]); info.per_replica_pools = {std::move(pool)}; addresses_with_failover.emplace_back(Addresses{address}); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index e35d31d2350..1ac7e776f9c 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -36,6 +36,13 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool fallback_to_stale_replicas_for_distributed_queries; + extern const SettingsUInt64 max_replica_delay_for_distributed_queries; + extern const SettingsBool prefer_localhost_replica; +} namespace ErrorCodes { @@ -68,7 +75,7 @@ ASTPtr rewriteSelectQuery( // are written into the query context and will be sent by the query pipeline. select_query.setExpression(ASTSelectQuery::Expression::SETTINGS, {}); - if (!context->getSettingsRef().allow_experimental_analyzer) + if (!context->getSettingsRef()[Setting::allow_experimental_analyzer]) { if (table_function_ptr) select_query.addTableFunction(table_function_ptr); @@ -165,7 +172,7 @@ void SelectStreamFactory::createForShardImpl( auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0) { Block shard_header; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze()); else shard_header = header; @@ -194,7 +201,7 @@ void SelectStreamFactory::createForShardImpl( // prefer_localhost_replica is not effective in case of parallel replicas // (1) prefer_localhost_replica is about choosing one replica on a shard // (2) parallel replica coordinator has own logic to choose replicas to read from - if (settings.prefer_localhost_replica && shard_info.isLocal() && !parallel_replicas_enabled) + if (settings[Setting::prefer_localhost_replica] && shard_info.isLocal() && !parallel_replicas_enabled) { StoragePtr main_table_storage; @@ -235,7 +242,7 @@ void SelectStreamFactory::createForShardImpl( return; } - const UInt64 max_allowed_delay = settings.max_replica_delay_for_distributed_queries; + const UInt64 max_allowed_delay = settings[Setting::max_replica_delay_for_distributed_queries]; if (!max_allowed_delay) { @@ -255,7 +262,7 @@ void SelectStreamFactory::createForShardImpl( ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); LOG_WARNING(getLogger("ClusterProxy::SelectStreamFactory"), "Local replica of shard {} is stale (delay: {}s.)", shard_info.shard_num, local_delay); - if (!settings.fallback_to_stale_replicas_for_distributed_queries) + if (!settings[Setting::fallback_to_stale_replicas_for_distributed_queries]) { if (shard_info.hasRemoteConnections()) { diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 1f854c41873..c1d301f068d 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -33,6 +33,38 @@ namespace DB { +namespace Setting +{ + extern const SettingsMap additional_table_filters; + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsUInt64 force_optimize_skip_unused_shards; + extern const SettingsUInt64 force_optimize_skip_unused_shards_nesting; + extern const SettingsUInt64 limit; + extern const SettingsLoadBalancing load_balancing; + extern const SettingsUInt64 max_concurrent_queries_for_user; + extern const SettingsUInt64 max_distributed_depth; + extern const SettingsSeconds max_execution_time; + extern const SettingsSeconds max_execution_time_leaf; + extern const SettingsUInt64 max_memory_usage_for_user; + extern const SettingsUInt64 max_network_bandwidth; + extern const SettingsUInt64 max_network_bytes; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsUInt64 offset; + extern const SettingsBool optimize_skip_unused_shards; + extern const SettingsUInt64 optimize_skip_unused_shards_nesting; + extern const SettingsBool optimize_skip_unused_shards_rewrite_in; + extern const SettingsString parallel_replicas_custom_key; + extern const SettingsParallelReplicasCustomKeyFilterType parallel_replicas_custom_key_filter_type; + extern const SettingsUInt64 parallel_replicas_custom_key_range_lower; + extern const SettingsUInt64 parallel_replicas_custom_key_range_upper; + extern const SettingsBool parallel_replicas_local_plan; + extern const SettingsMilliseconds queue_max_wait_ms; + extern const SettingsBool skip_unavailable_shards; + extern const SettingsOverflowMode timeout_overflow_mode; + extern const SettingsOverflowMode timeout_overflow_mode_leaf; + extern const SettingsBool use_hedged_requests; +} namespace ErrorCodes { @@ -55,8 +87,8 @@ ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & cluster, const DistributedSettings * distributed_settings) { ClientInfo new_client_info = context->getClientInfo(); - Settings new_settings = settings; - new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); + Settings new_settings {settings}; + new_settings[Setting::queue_max_wait_ms] = Cluster::saturate(new_settings[Setting::queue_max_wait_ms], settings[Setting::max_execution_time]); /// In case of interserver mode we should reset initial_user for remote() function to use passed user from the query. if (is_remote_function) @@ -72,69 +104,70 @@ ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & cluster, if (!interserver_mode) { /// Does not matter on remote servers, because queries are sent under different user. - new_settings.max_concurrent_queries_for_user = 0; - new_settings.max_memory_usage_for_user = 0; + new_settings[Setting::max_concurrent_queries_for_user] = 0; + new_settings[Setting::max_memory_usage_for_user] = 0; /// Set as unchanged to avoid sending to remote server. - new_settings.max_concurrent_queries_for_user.changed = false; - new_settings.max_memory_usage_for_user.changed = false; + new_settings[Setting::max_concurrent_queries_for_user].changed = false; + new_settings[Setting::max_memory_usage_for_user].changed = false; } - if (settings.force_optimize_skip_unused_shards_nesting && settings.force_optimize_skip_unused_shards) + if (settings[Setting::force_optimize_skip_unused_shards_nesting] && settings[Setting::force_optimize_skip_unused_shards]) { - if (new_settings.force_optimize_skip_unused_shards_nesting == 1) + if (new_settings[Setting::force_optimize_skip_unused_shards_nesting] == 1) { - new_settings.force_optimize_skip_unused_shards = false; - new_settings.force_optimize_skip_unused_shards.changed = false; + new_settings[Setting::force_optimize_skip_unused_shards] = false; + new_settings[Setting::force_optimize_skip_unused_shards].changed = false; if (log) LOG_TRACE(log, "Disabling force_optimize_skip_unused_shards for nested queries (force_optimize_skip_unused_shards_nesting exceeded)"); } else { - --new_settings.force_optimize_skip_unused_shards_nesting.value; - new_settings.force_optimize_skip_unused_shards_nesting.changed = true; + --new_settings[Setting::force_optimize_skip_unused_shards_nesting].value; + new_settings[Setting::force_optimize_skip_unused_shards_nesting].changed = true; if (log) - LOG_TRACE(log, "force_optimize_skip_unused_shards_nesting is now {}", new_settings.force_optimize_skip_unused_shards_nesting); + LOG_TRACE( + log, "force_optimize_skip_unused_shards_nesting is now {}", new_settings[Setting::force_optimize_skip_unused_shards_nesting]); } } - if (settings.optimize_skip_unused_shards_nesting && settings.optimize_skip_unused_shards) + if (settings[Setting::optimize_skip_unused_shards_nesting] && settings[Setting::optimize_skip_unused_shards]) { - if (new_settings.optimize_skip_unused_shards_nesting == 1) + if (new_settings[Setting::optimize_skip_unused_shards_nesting] == 1) { - new_settings.optimize_skip_unused_shards = false; - new_settings.optimize_skip_unused_shards.changed = false; + new_settings[Setting::optimize_skip_unused_shards] = false; + new_settings[Setting::optimize_skip_unused_shards].changed = false; if (log) LOG_TRACE(log, "Disabling optimize_skip_unused_shards for nested queries (optimize_skip_unused_shards_nesting exceeded)"); } else { - --new_settings.optimize_skip_unused_shards_nesting.value; - new_settings.optimize_skip_unused_shards_nesting.changed = true; + --new_settings[Setting::optimize_skip_unused_shards_nesting].value; + new_settings[Setting::optimize_skip_unused_shards_nesting].changed = true; if (log) - LOG_TRACE(log, "optimize_skip_unused_shards_nesting is now {}", new_settings.optimize_skip_unused_shards_nesting); + LOG_TRACE(log, "optimize_skip_unused_shards_nesting is now {}", new_settings[Setting::optimize_skip_unused_shards_nesting]); } } - if (!settings.skip_unavailable_shards.changed && distributed_settings) + if (!settings[Setting::skip_unavailable_shards].changed && distributed_settings) { - new_settings.skip_unavailable_shards = distributed_settings->skip_unavailable_shards.value; - new_settings.skip_unavailable_shards.changed = true; + new_settings[Setting::skip_unavailable_shards] = distributed_settings->skip_unavailable_shards.value; + new_settings[Setting::skip_unavailable_shards].changed = true; } - if (settings.offset) + if (settings[Setting::offset]) { - new_settings.offset = 0; - new_settings.offset.changed = false; + new_settings[Setting::offset] = 0; + new_settings[Setting::offset].changed = false; } - if (settings.limit) + if (settings[Setting::limit]) { - new_settings.limit = 0; - new_settings.limit.changed = false; + new_settings[Setting::limit] = 0; + new_settings[Setting::limit].changed = false; } /// Setting additional_table_filters may be applied to Distributed table. @@ -148,7 +181,7 @@ ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & cluster, Tuple tuple; tuple.push_back(main_table.getShortName()); tuple.push_back(queryToString(additional_filter_ast)); - new_settings.additional_table_filters.value.push_back(std::move(tuple)); + new_settings[Setting::additional_table_filters].value.push_back(std::move(tuple)); } /// disable parallel replicas if cluster contains only shards with 1 replica @@ -164,22 +197,22 @@ ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & cluster, } } if (disable_parallel_replicas) - new_settings.allow_experimental_parallel_reading_from_replicas = 0; + new_settings[Setting::allow_experimental_parallel_reading_from_replicas] = 0; } - if (settings.max_execution_time_leaf.value > 0) + if (settings[Setting::max_execution_time_leaf].value > 0) { /// Replace 'max_execution_time' of this sub-query with 'max_execution_time_leaf' and 'timeout_overflow_mode' /// with 'timeout_overflow_mode_leaf' - new_settings.max_execution_time = settings.max_execution_time_leaf; - new_settings.timeout_overflow_mode = settings.timeout_overflow_mode_leaf; + new_settings[Setting::max_execution_time] = settings[Setting::max_execution_time_leaf]; + new_settings[Setting::timeout_overflow_mode] = settings[Setting::timeout_overflow_mode_leaf]; } /// in case of parallel replicas custom key use round robing load balancing /// so custom key partitions will be spread over nodes in round-robin fashion - if (context->canUseParallelReplicasCustomKeyForCluster(cluster) && !settings.load_balancing.changed) + if (context->canUseParallelReplicasCustomKeyForCluster(cluster) && !settings[Setting::load_balancing].changed) { - new_settings.load_balancing = LoadBalancing::ROUND_ROBIN; + new_settings[Setting::load_balancing] = LoadBalancing::ROUND_ROBIN; } auto new_context = Context::createCopy(context); @@ -215,13 +248,13 @@ static ThrottlerPtr getThrottler(const ContextPtr & context) /// Network bandwidth limit, if needed. ThrottlerPtr throttler; - if (settings.max_network_bandwidth || settings.max_network_bytes) + if (settings[Setting::max_network_bandwidth] || settings[Setting::max_network_bytes]) { throttler = std::make_shared( - settings.max_network_bandwidth, - settings.max_network_bytes, - "Limit for bytes to send or receive over network exceeded.", - user_level_throttler); + settings[Setting::max_network_bandwidth], + settings[Setting::max_network_bytes], + "Limit for bytes to send or receive over network exceeded.", + user_level_throttler); } else throttler = user_level_throttler; @@ -236,15 +269,15 @@ getShardFilterGeneratorForCustomKey(const Cluster & cluster, ContextPtr context, return {}; const auto & settings = context->getSettingsRef(); - auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *context); + auto custom_key_ast = parseCustomKeyForTable(settings[Setting::parallel_replicas_custom_key], *context); if (custom_key_ast == nullptr) return {}; return [my_custom_key_ast = std::move(custom_key_ast), column_description = columns, - custom_key_type = settings.parallel_replicas_custom_key_filter_type.value, - custom_key_range_lower = settings.parallel_replicas_custom_key_range_lower.value, - custom_key_range_upper = settings.parallel_replicas_custom_key_range_upper.value, + custom_key_type = settings[Setting::parallel_replicas_custom_key_filter_type].value, + custom_key_range_lower = settings[Setting::parallel_replicas_custom_key_range_lower].value, + custom_key_range_upper = settings[Setting::parallel_replicas_custom_key_range_upper].value, query_context = context, replica_count = cluster.getShardsInfo().front().per_replica_pools.size()](uint64_t replica_num) -> ASTPtr { @@ -277,7 +310,7 @@ void executeQuery( { const Settings & settings = context->getSettingsRef(); - if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth) + if (settings[Setting::max_distributed_depth] && context->getClientInfo().distributed_depth >= settings[Setting::max_distributed_depth]) throw Exception(ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH, "Maximum distributed depth exceeded"); const ClusterPtr & not_optimized_cluster = query_info.cluster; @@ -288,9 +321,9 @@ void executeQuery( auto cluster = query_info.getCluster(); auto new_context = updateSettingsAndClientInfoForCluster(*cluster, is_remote_function, context, settings, main_table, query_info.additional_filter_ast, log, &distributed_settings); - if (context->getSettingsRef().allow_experimental_parallel_reading_from_replicas - && context->getSettingsRef().allow_experimental_parallel_reading_from_replicas.value - != new_context->getSettingsRef().allow_experimental_parallel_reading_from_replicas.value) + if (context->getSettingsRef()[Setting::allow_experimental_parallel_reading_from_replicas] + && context->getSettingsRef()[Setting::allow_experimental_parallel_reading_from_replicas].value + != new_context->getSettingsRef()[Setting::allow_experimental_parallel_reading_from_replicas].value) { LOG_TRACE( log, @@ -302,17 +335,14 @@ void executeQuery( const size_t shards = cluster->getShardCount(); - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i) { const auto & shard_info = cluster->getShardsInfo()[i]; auto query_for_shard = query_info.query_tree->clone(); - if (sharding_key_expr && - query_info.optimized_cluster && - settings.optimize_skip_unused_shards_rewrite_in && - shards > 1 && + if (sharding_key_expr && query_info.optimized_cluster && settings[Setting::optimize_skip_unused_shards_rewrite_in] && shards > 1 && /// TODO: support composite sharding key sharding_key_expr->getRequiredColumns().size() == 1) { @@ -350,10 +380,7 @@ void executeQuery( const auto & shard_info = cluster->getShardsInfo()[i]; ASTPtr query_ast_for_shard = query_info.query->clone(); - if (sharding_key_expr && - query_info.optimized_cluster && - settings.optimize_skip_unused_shards_rewrite_in && - shards > 1 && + if (sharding_key_expr && query_info.optimized_cluster && settings[Setting::optimize_skip_unused_shards_rewrite_in] && shards > 1 && /// TODO: support composite sharding key sharding_key_expr->getRequiredColumns().size() == 1) { @@ -456,9 +483,9 @@ void executeQueryWithParallelReplicas( auto new_context = Context::createCopy(context); /// check hedged connections setting - if (settings.use_hedged_requests.value) + if (settings[Setting::use_hedged_requests].value) { - if (settings.use_hedged_requests.changed) + if (settings[Setting::use_hedged_requests].changed) { LOG_WARNING( getLogger("executeQueryWithParallelReplicas"), @@ -520,14 +547,14 @@ void executeQueryWithParallelReplicas( } const auto & shard = new_cluster->getShardsInfo().at(0); - size_t max_replicas_to_use = settings.max_parallel_replicas; + size_t max_replicas_to_use = settings[Setting::max_parallel_replicas]; if (max_replicas_to_use > shard.getAllNodeCount()) { LOG_INFO( getLogger("ReadFromParallelRemoteReplicasStep"), "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). " "Will use the latter number to execute the query.", - settings.max_parallel_replicas, + settings[Setting::max_parallel_replicas], shard.getAllNodeCount()); max_replicas_to_use = shard.getAllNodeCount(); } @@ -558,7 +585,7 @@ void executeQueryWithParallelReplicas( pools_to_use.emplace_back(std::move(pool.pool)); /// do not build local plan for distributed queries for now (address it later) - if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) + if (settings[Setting::allow_experimental_analyzer] && settings[Setting::parallel_replicas_local_plan] && !shard_num) { /// find local replica index in pool std::optional local_replica_index; @@ -712,7 +739,7 @@ void executeQueryWithParallelReplicasCustomKey( /// Return directly (with correct header) if no shard to query. if (query_info.getCluster()->getShardsInfo().empty()) { - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) return; Pipe pipe(std::make_shared(header)); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 311fd094706..a22cf4b0ff1 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -177,6 +178,66 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsMilliseconds async_insert_poll_timeout_ms; + extern const SettingsBool azure_allow_parallel_part_upload; + extern const SettingsUInt64 backup_threads; + extern const SettingsString cluster_for_parallel_replicas; + extern const SettingsBool enable_filesystem_cache; + extern const SettingsBool enable_filesystem_cache_log; + extern const SettingsBool enable_filesystem_cache_on_write_operations; + extern const SettingsBool enable_filesystem_read_prefetches_log; + extern const SettingsBool enable_blob_storage_log; + extern const SettingsUInt64 filesystem_cache_max_download_size; + extern const SettingsUInt64 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; + extern const SettingsUInt64 filesystem_cache_segments_batch_size; + extern const SettingsBool http_make_head_request; + extern const SettingsUInt64 http_max_fields; + extern const SettingsUInt64 http_max_field_name_size; + extern const SettingsUInt64 http_max_field_value_size; + extern const SettingsUInt64 http_max_tries; + extern const SettingsUInt64 http_max_uri_size; + extern const SettingsSeconds http_receive_timeout; + extern const SettingsUInt64 http_retry_initial_backoff_ms; + extern const SettingsUInt64 http_retry_max_backoff_ms; + extern const SettingsSeconds http_send_timeout; + extern const SettingsBool http_skip_not_found_url_for_globs; + extern const SettingsUInt64 hsts_max_age; + extern const SettingsString local_filesystem_read_method; + extern const SettingsBool local_filesystem_read_prefetch; + extern const SettingsBool load_marks_asynchronously; + extern const SettingsUInt64 max_backup_bandwidth; + extern const SettingsUInt64 max_local_read_bandwidth; + extern const SettingsUInt64 max_local_write_bandwidth; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsUInt64 max_read_buffer_size; + extern const SettingsUInt64 max_read_buffer_size_local_fs; + extern const SettingsUInt64 max_read_buffer_size_remote_fs; + extern const SettingsUInt64 max_remote_read_network_bandwidth; + extern const SettingsUInt64 max_remote_write_network_bandwidth; + extern const SettingsUInt64 min_bytes_to_use_direct_io; + extern const SettingsUInt64 min_bytes_to_use_mmap_io; + extern const SettingsBool page_cache_inject_eviction; + extern const SettingsString parallel_replicas_custom_key; + extern const SettingsUInt64 prefetch_buffer_size; + extern const SettingsBool read_from_filesystem_cache_if_exists_otherwise_bypass_cache; + extern const SettingsBool read_from_page_cache_if_exists_otherwise_bypass_cache; + extern const SettingsInt64 read_priority; + extern const SettingsUInt64 restore_threads; + extern const SettingsString remote_filesystem_read_method; + extern const SettingsBool remote_filesystem_read_prefetch; + extern const SettingsUInt64 remote_fs_read_max_backoff_ms; + extern const SettingsUInt64 remote_fs_read_backoff_max_tries; + extern const SettingsUInt64 remote_read_min_bytes_for_seek; + extern const SettingsBool throw_on_error_from_cache_on_write_operations; + extern const SettingsBool skip_download_if_exceeds_query_cache; + extern const SettingsBool s3_allow_parallel_part_upload; + extern const SettingsBool use_page_cache_for_disks_without_file_cache; + extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; + extern const SettingsString workload; +} namespace ErrorCodes { @@ -1039,12 +1100,7 @@ Strings Context::getWarnings() const common_warnings.emplace_back(fmt::format("The number of active parts is more than {}", shared->max_part_num_to_warn)); } /// Make setting's name ordered - std::set obsolete_settings; - for (const auto & setting : *settings) - { - if (setting.isValueChanged() && setting.isObsolete()) - obsolete_settings.emplace(setting.getName()); - } + auto obsolete_settings = settings->getChangedAndObsoleteNames(); if (!obsolete_settings.empty()) { @@ -1055,7 +1111,9 @@ Strings Context::getWarnings() const for (const auto & setting : obsolete_settings) { res += first ? "" : ", "; - res += "'" + setting + "'"; + res += "'"; + res += setting; + res += "'"; first = false; } res = res + "]" + (single_element ? " is" : " are") @@ -1555,7 +1613,8 @@ std::shared_ptr Context::getAccess() const /// If setUserID() was never called then this must be the global context with the full access. bool full_access = !user_id; - return ContextAccessParams{user_id, full_access, /* use_default_roles= */ false, current_roles, *settings, current_database, client_info}; + return ContextAccessParams{ + user_id, full_access, /* use_default_roles= */ false, current_roles, *settings, current_database, client_info}; }; /// Check if the current access rights are still valid, otherwise get parameters for recalculating access rights. @@ -1685,7 +1744,7 @@ ClassifierPtr Context::getWorkloadClassifier() const std::lock_guard lock(mutex); // NOTE: Workload cannot be changed after query start, and getWorkloadClassifier() should not be called before proper `workload` is set if (!classifier) - classifier = getResourceManager()->acquire(getSettingsRef().workload); + classifier = getResourceManager()->acquire(getSettingsRef()[Setting::workload]); return classifier; } @@ -2062,8 +2121,10 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const throw; } - uint64_t use_structure_from_insertion_table_in_table_functions = getSettingsRef().use_structure_from_insertion_table_in_table_functions; - if (select_query_hint && use_structure_from_insertion_table_in_table_functions && table_function_ptr->needStructureHint() && hasInsertionTable()) + uint64_t use_structure_from_insertion_table_in_table_functions + = getSettingsRef()[Setting::use_structure_from_insertion_table_in_table_functions]; + if (select_query_hint && use_structure_from_insertion_table_in_table_functions && table_function_ptr->needStructureHint() + && hasInsertionTable()) { const auto & insert_columns = DatabaseCatalog::instance() .getTable(getInsertionTable(), shared_from_this()) @@ -2191,8 +2252,8 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const if (!structure_hint.empty()) table_function_ptr->setStructureHint(structure_hint); - - } else if (use_structure_from_insertion_table_in_table_functions == 1) + } + else if (use_structure_from_insertion_table_in_table_functions == 1) throw Exception(ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, "Number of columns in insert table less than required by SELECT expression."); } } @@ -2679,14 +2740,15 @@ void Context::makeQueryContextForMerge(const MergeTreeSettings & merge_tree_sett { makeQueryContext(); classifier.reset(); // It is assumed that there are no active queries running using this classifier, otherwise this will lead to crashes - settings->workload = merge_tree_settings.merge_workload.value.empty() ? getMergeWorkload() : merge_tree_settings.merge_workload; + (*settings)[Setting::workload] = merge_tree_settings.merge_workload.value.empty() ? getMergeWorkload() : merge_tree_settings.merge_workload; } void Context::makeQueryContextForMutate(const MergeTreeSettings & merge_tree_settings) { makeQueryContext(); classifier.reset(); // It is assumed that there are no active queries running using this classifier, otherwise this will lead to crashes - settings->workload = merge_tree_settings.mutation_workload.value.empty() ? getMutationWorkload() : merge_tree_settings.mutation_workload; + (*settings)[Setting::workload] + = merge_tree_settings.mutation_workload.value.empty() ? getMutationWorkload() : merge_tree_settings.mutation_workload; } void Context::makeSessionContext() @@ -2935,8 +2997,8 @@ BackupsWorker & Context::getBackupsWorker() const callOnce(shared->backups_worker_initialized, [&] { const auto & config = getConfigRef(); const auto & settings_ref = getSettingsRef(); - UInt64 backup_threads = config.getUInt64("backup_threads", settings_ref.backup_threads); - UInt64 restore_threads = config.getUInt64("restore_threads", settings_ref.restore_threads); + UInt64 backup_threads = config.getUInt64("backup_threads", settings_ref[Setting::backup_threads]); + UInt64 restore_threads = config.getUInt64("restore_threads", settings_ref[Setting::restore_threads]); shared->backups_worker.emplace(getGlobalContext(), backup_threads, restore_threads); }); @@ -3413,7 +3475,7 @@ ThrottlerPtr Context::getReplicatedSendsThrottler() const ThrottlerPtr Context::getRemoteReadThrottler() const { ThrottlerPtr throttler = shared->remote_read_throttler; - if (auto bandwidth = getSettingsRef().max_remote_read_network_bandwidth) + if (auto bandwidth = getSettingsRef()[Setting::max_remote_read_network_bandwidth]) { std::lock_guard lock(mutex); if (!remote_read_query_throttler) @@ -3426,7 +3488,7 @@ ThrottlerPtr Context::getRemoteReadThrottler() const ThrottlerPtr Context::getRemoteWriteThrottler() const { ThrottlerPtr throttler = shared->remote_write_throttler; - if (auto bandwidth = getSettingsRef().max_remote_write_network_bandwidth) + if (auto bandwidth = getSettingsRef()[Setting::max_remote_write_network_bandwidth]) { std::lock_guard lock(mutex); if (!remote_write_query_throttler) @@ -3439,7 +3501,7 @@ ThrottlerPtr Context::getRemoteWriteThrottler() const ThrottlerPtr Context::getLocalReadThrottler() const { ThrottlerPtr throttler = shared->local_read_throttler; - if (auto bandwidth = getSettingsRef().max_local_read_bandwidth) + if (auto bandwidth = getSettingsRef()[Setting::max_local_read_bandwidth]) { std::lock_guard lock(mutex); if (!local_read_query_throttler) @@ -3452,7 +3514,7 @@ ThrottlerPtr Context::getLocalReadThrottler() const ThrottlerPtr Context::getLocalWriteThrottler() const { ThrottlerPtr throttler = shared->local_write_throttler; - if (auto bandwidth = getSettingsRef().max_local_write_bandwidth) + if (auto bandwidth = getSettingsRef()[Setting::max_local_write_bandwidth]) { std::lock_guard lock(mutex); if (!local_write_query_throttler) @@ -3465,7 +3527,7 @@ ThrottlerPtr Context::getLocalWriteThrottler() const ThrottlerPtr Context::getBackupsThrottler() const { ThrottlerPtr throttler = shared->backups_server_throttler; - if (auto bandwidth = getSettingsRef().max_backup_bandwidth) + if (auto bandwidth = getSettingsRef()[Setting::max_backup_bandwidth]) { std::lock_guard lock(mutex); if (!backups_query_throttler) @@ -4330,9 +4392,9 @@ std::shared_ptr Context::getBackupLog() const std::shared_ptr Context::getBlobStorageLog() const { - bool enable_blob_storage_log = settings->enable_blob_storage_log; + bool enable_blob_storage_log = getSettingsRef()[Setting::enable_blob_storage_log]; if (hasQueryContext()) - enable_blob_storage_log = getQueryContext()->getSettingsRef().enable_blob_storage_log; + enable_blob_storage_log = getQueryContext()->getSettingsRef()[Setting::enable_blob_storage_log]; if (!enable_blob_storage_log) return {}; @@ -5383,7 +5445,7 @@ void Context::setAsynchronousInsertQueue(const std::shared_ptrmutex); - if (std::chrono::milliseconds(settings->async_insert_poll_timeout_ms) == std::chrono::milliseconds::zero()) + if (std::chrono::milliseconds(getSettingsRef()[Setting::async_insert_poll_timeout_ms]) == std::chrono::milliseconds::zero()) throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting async_insert_poll_timeout_ms can't be zero"); shared->async_insert_queue = ptr; @@ -5529,70 +5591,73 @@ ThreadPool & Context::getThreadPoolWriter() const ReadSettings Context::getReadSettings() const { ReadSettings res; + const auto & settings_ref = getSettingsRef(); - std::string_view read_method_str = settings->local_filesystem_read_method.value; + std::string_view read_method_str = getSettingsRef()[Setting::local_filesystem_read_method].value; if (auto opt_method = magic_enum::enum_cast(read_method_str)) res.local_fs_method = *opt_method; else throw Exception(ErrorCodes::UNKNOWN_READ_METHOD, "Unknown read method '{}' for local filesystem", read_method_str); - read_method_str = settings->remote_filesystem_read_method.value; + read_method_str = getSettingsRef()[Setting::remote_filesystem_read_method].value; if (auto opt_method = magic_enum::enum_cast(read_method_str)) res.remote_fs_method = *opt_method; else throw Exception(ErrorCodes::UNKNOWN_READ_METHOD, "Unknown read method '{}' for remote filesystem", read_method_str); - res.local_fs_prefetch = settings->local_filesystem_read_prefetch; - res.remote_fs_prefetch = settings->remote_filesystem_read_prefetch; + res.local_fs_prefetch = settings_ref[Setting::local_filesystem_read_prefetch]; + res.remote_fs_prefetch = settings_ref[Setting::remote_filesystem_read_prefetch]; - res.load_marks_asynchronously = settings->load_marks_asynchronously; + res.load_marks_asynchronously = settings_ref[Setting::load_marks_asynchronously]; - res.enable_filesystem_read_prefetches_log = settings->enable_filesystem_read_prefetches_log; + res.enable_filesystem_read_prefetches_log = settings_ref[Setting::enable_filesystem_read_prefetches_log]; - res.remote_fs_read_max_backoff_ms = settings->remote_fs_read_max_backoff_ms; - res.remote_fs_read_backoff_max_tries = settings->remote_fs_read_backoff_max_tries; - res.enable_filesystem_cache = settings->enable_filesystem_cache; - res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings->read_from_filesystem_cache_if_exists_otherwise_bypass_cache; - res.enable_filesystem_cache_log = settings->enable_filesystem_cache_log; - res.filesystem_cache_segments_batch_size = settings->filesystem_cache_segments_batch_size; - res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings->filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; + res.remote_fs_read_max_backoff_ms = settings_ref[Setting::remote_fs_read_max_backoff_ms]; + res.remote_fs_read_backoff_max_tries = settings_ref[Setting::remote_fs_read_backoff_max_tries]; + res.enable_filesystem_cache = settings_ref[Setting::enable_filesystem_cache]; + res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache + = settings_ref[Setting::read_from_filesystem_cache_if_exists_otherwise_bypass_cache]; + res.enable_filesystem_cache_log = settings_ref[Setting::enable_filesystem_cache_log]; + res.filesystem_cache_segments_batch_size = settings_ref[Setting::filesystem_cache_segments_batch_size]; + res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds + = settings_ref[Setting::filesystem_cache_reserve_space_wait_lock_timeout_milliseconds]; - res.filesystem_cache_max_download_size = settings->filesystem_cache_max_download_size; - res.skip_download_if_exceeds_query_cache = settings->skip_download_if_exceeds_query_cache; + res.filesystem_cache_max_download_size = settings_ref[Setting::filesystem_cache_max_download_size]; + res.skip_download_if_exceeds_query_cache = settings_ref[Setting::skip_download_if_exceeds_query_cache]; res.page_cache = getPageCache(); - res.use_page_cache_for_disks_without_file_cache = settings->use_page_cache_for_disks_without_file_cache; - res.read_from_page_cache_if_exists_otherwise_bypass_cache = settings->read_from_page_cache_if_exists_otherwise_bypass_cache; - res.page_cache_inject_eviction = settings->page_cache_inject_eviction; + res.use_page_cache_for_disks_without_file_cache = settings_ref[Setting::use_page_cache_for_disks_without_file_cache]; + res.read_from_page_cache_if_exists_otherwise_bypass_cache = settings_ref[Setting::read_from_page_cache_if_exists_otherwise_bypass_cache]; + res.page_cache_inject_eviction = settings_ref[Setting::page_cache_inject_eviction]; - res.remote_read_min_bytes_for_seek = settings->remote_read_min_bytes_for_seek; + res.remote_read_min_bytes_for_seek = getSettingsRef()[Setting::remote_read_min_bytes_for_seek]; /// Zero read buffer will not make progress. - if (!settings->max_read_buffer_size) + if (!getSettingsRef()[Setting::max_read_buffer_size]) { - throw Exception(ErrorCodes::INVALID_SETTING_VALUE, - "Invalid value '{}' for max_read_buffer_size", settings->max_read_buffer_size); + throw Exception( + ErrorCodes::INVALID_SETTING_VALUE, "Invalid value '{}' for max_read_buffer_size", getSettingsRef()[Setting::max_read_buffer_size]); } res.local_fs_buffer_size - = settings->max_read_buffer_size_local_fs ? settings->max_read_buffer_size_local_fs : settings->max_read_buffer_size; + = settings_ref[Setting::max_read_buffer_size_local_fs] ? settings_ref[Setting::max_read_buffer_size_local_fs] : settings_ref[Setting::max_read_buffer_size]; res.remote_fs_buffer_size - = settings->max_read_buffer_size_remote_fs ? settings->max_read_buffer_size_remote_fs : settings->max_read_buffer_size; - res.prefetch_buffer_size = settings->prefetch_buffer_size; - res.direct_io_threshold = settings->min_bytes_to_use_direct_io; - res.mmap_threshold = settings->min_bytes_to_use_mmap_io; - res.priority = Priority{settings->read_priority}; + = settings_ref[Setting::max_read_buffer_size_remote_fs] ? settings_ref[Setting::max_read_buffer_size_remote_fs] : settings_ref[Setting::max_read_buffer_size]; + res.prefetch_buffer_size = settings_ref[Setting::prefetch_buffer_size]; + res.direct_io_threshold = settings_ref[Setting::min_bytes_to_use_direct_io]; + res.mmap_threshold = settings_ref[Setting::min_bytes_to_use_mmap_io]; + res.priority = Priority{settings_ref[Setting::read_priority]}; res.remote_throttler = getRemoteReadThrottler(); res.local_throttler = getLocalReadThrottler(); - res.http_max_tries = settings->http_max_tries; - res.http_retry_initial_backoff_ms = settings->http_retry_initial_backoff_ms; - res.http_retry_max_backoff_ms = settings->http_retry_max_backoff_ms; - res.http_skip_not_found_url_for_globs = settings->http_skip_not_found_url_for_globs; - res.http_make_head_request = settings->http_make_head_request; + res.http_max_tries = settings_ref[Setting::http_max_tries]; + res.http_retry_initial_backoff_ms = settings_ref[Setting::http_retry_initial_backoff_ms]; + res.http_retry_max_backoff_ms = settings_ref[Setting::http_retry_max_backoff_ms]; + res.http_skip_not_found_url_for_globs = settings_ref[Setting::http_skip_not_found_url_for_globs]; + res.http_make_head_request = settings_ref[Setting::http_make_head_request]; res.mmap_cache = getMMappedFileCache().get(); @@ -5602,14 +5667,16 @@ ReadSettings Context::getReadSettings() const WriteSettings Context::getWriteSettings() const { WriteSettings res; + const auto & settings_ref = getSettingsRef(); - res.enable_filesystem_cache_on_write_operations = settings->enable_filesystem_cache_on_write_operations; - res.enable_filesystem_cache_log = settings->enable_filesystem_cache_log; - res.throw_on_error_from_cache = settings->throw_on_error_from_cache_on_write_operations; - res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds = settings->filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; + res.enable_filesystem_cache_on_write_operations = settings_ref[Setting::enable_filesystem_cache_on_write_operations]; + res.enable_filesystem_cache_log = settings_ref[Setting::enable_filesystem_cache_log]; + res.throw_on_error_from_cache = settings_ref[Setting::throw_on_error_from_cache_on_write_operations]; + res.filesystem_cache_reserve_space_wait_lock_timeout_milliseconds + = settings_ref[Setting::filesystem_cache_reserve_space_wait_lock_timeout_milliseconds]; - res.s3_allow_parallel_part_upload = settings->s3_allow_parallel_part_upload; - res.azure_allow_parallel_part_upload = settings->azure_allow_parallel_part_upload; + res.s3_allow_parallel_part_upload = settings_ref[Setting::s3_allow_parallel_part_upload]; + res.azure_allow_parallel_part_upload = settings_ref[Setting::azure_allow_parallel_part_upload]; res.remote_throttler = getRemoteWriteThrottler(); res.local_throttler = getLocalWriteThrottler(); @@ -5630,10 +5697,10 @@ Context::ParallelReplicasMode Context::getParallelReplicasMode() const const auto & settings_ref = getSettingsRef(); using enum Context::ParallelReplicasMode; - if (!settings_ref.parallel_replicas_custom_key.value.empty()) + if (!settings_ref[Setting::parallel_replicas_custom_key].value.empty()) return CUSTOM_KEY; - if (settings_ref.allow_experimental_parallel_reading_from_replicas > 0) + if (settings_ref[Setting::allow_experimental_parallel_reading_from_replicas] > 0) return READ_TASKS; return SAMPLE_KEY; @@ -5642,7 +5709,7 @@ Context::ParallelReplicasMode Context::getParallelReplicasMode() const bool Context::canUseTaskBasedParallelReplicas() const { const auto & settings_ref = getSettingsRef(); - return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings_ref.max_parallel_replicas > 1; + return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings_ref[Setting::max_parallel_replicas] > 1; } bool Context::canUseParallelReplicasOnInitiator() const @@ -5657,7 +5724,7 @@ bool Context::canUseParallelReplicasOnFollower() const bool Context::canUseParallelReplicasCustomKey() const { - return settings->max_parallel_replicas > 1 && getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY; + return getSettingsRef()[Setting::max_parallel_replicas] > 1 && getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY; } bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) const @@ -5667,7 +5734,7 @@ bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) bool Context::canUseOffsetParallelReplicas() const { - return offset_parallel_replicas_enabled && settings->max_parallel_replicas > 1 + return offset_parallel_replicas_enabled && getSettingsRef()[Setting::max_parallel_replicas] > 1 && getParallelReplicasMode() != Context::ParallelReplicasMode::READ_TASKS; } @@ -5678,14 +5745,15 @@ void Context::disableOffsetParallelReplicas() ClusterPtr Context::getClusterForParallelReplicas() const { + const auto & settings_ref = getSettingsRef(); /// check cluster for parallel replicas - if (settings->cluster_for_parallel_replicas.value.empty()) + if (settings_ref[Setting::cluster_for_parallel_replicas].value.empty()) throw Exception( ErrorCodes::CLUSTER_DOESNT_EXIST, "Reading in parallel from replicas is enabled but cluster to execute query is not provided. Please set " "'cluster_for_parallel_replicas' setting"); - return getCluster(settings->cluster_for_parallel_replicas); + return getCluster(settings_ref[Setting::cluster_for_parallel_replicas]); } void Context::setPreparedSetsCache(const PreparedSetsCachePtr & cache) @@ -5715,37 +5783,37 @@ const ServerSettings & Context::getServerSettings() const uint64_t HTTPContext::getMaxHstsAge() const { - return context->getSettingsRef().hsts_max_age; + return context->getSettingsRef()[Setting::hsts_max_age]; } uint64_t HTTPContext::getMaxUriSize() const { - return context->getSettingsRef().http_max_uri_size; + return context->getSettingsRef()[Setting::http_max_uri_size]; } uint64_t HTTPContext::getMaxFields() const { -return context->getSettingsRef().http_max_fields; + return context->getSettingsRef()[Setting::http_max_fields]; } uint64_t HTTPContext::getMaxFieldNameSize() const { -return context->getSettingsRef().http_max_field_name_size; + return context->getSettingsRef()[Setting::http_max_field_name_size]; } uint64_t HTTPContext::getMaxFieldValueSize() const { -return context->getSettingsRef().http_max_field_value_size; + return context->getSettingsRef()[Setting::http_max_field_value_size]; } Poco::Timespan HTTPContext::getReceiveTimeout() const { -return context->getSettingsRef().http_receive_timeout; + return context->getSettingsRef()[Setting::http_receive_timeout]; } Poco::Timespan HTTPContext::getSendTimeout() const { -return context->getSettingsRef().http_send_timeout; + return context->getSettingsRef()[Setting::http_send_timeout]; } } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 0daef2243aa..4d09c206797 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1,29 +1,23 @@ #pragma once #include -#include #include -#include -#include #include #include #include #include #include -#include #include -#include -#include #include #include #include +#include #include #include #include #include #include #include -#include #include #include "config.h" @@ -34,7 +28,11 @@ #include -namespace Poco::Net { class IPAddress; } +namespace Poco::Net +{ +class IPAddress; +class SocketAddress; +} namespace zkutil { class ZooKeeper; @@ -134,10 +132,11 @@ class ICompressionCodec; class AccessControl; class GSSAcceptorContext; struct Settings; +struct SettingChange; +class SettingsChanges; struct SettingsConstraintsAndProfileIDs; class SettingsProfileElements; class RemoteHostFilter; -struct StorageID; class IDisk; using DiskPtr = std::shared_ptr; class DiskSelector; @@ -152,6 +151,8 @@ class ServerType; template class MergeTreeBackgroundExecutor; class AsyncLoader; +class HTTPHeaderFilter; +struct AsyncReadCounters; struct ICgroupsReader; struct TemporaryTableHolder; diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 6e08dd5e2cc..f70241373f9 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -22,6 +22,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsUInt64 distributed_ddl_entry_format_version; + extern const SettingsUInt64 log_queries_cut_to_length; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_query_size; +} + namespace ErrorCodes { extern const int UNKNOWN_FORMAT_VERSION; @@ -70,7 +80,7 @@ void DDLLogEntry::assertVersion() const void DDLLogEntry::setSettingsIfRequired(ContextPtr context) { - version = context->getSettingsRef().distributed_ddl_entry_format_version; + version = context->getSettingsRef()[Setting::distributed_ddl_entry_format_version]; if (version <= 0 || version > DDL_ENTRY_FORMAT_MAX_VERSION) throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unknown distributed_ddl_entry_format_version: {}." "Maximum supported version is {}.", version, DDL_ENTRY_FORMAT_MAX_VERSION); @@ -157,7 +167,8 @@ void DDLLogEntry::parse(const String & data) ParserSetQuery parser{true}; constexpr UInt64 max_depth = 16; constexpr UInt64 max_backtracks = DBMS_DEFAULT_MAX_PARSER_BACKTRACKS; - ASTPtr settings_ast = parseQuery(parser, settings_str, Context::getGlobalContextInstance()->getSettingsRef().max_query_size, max_depth, max_backtracks); + ASTPtr settings_ast = parseQuery( + parser, settings_str, Context::getGlobalContextInstance()->getSettingsRef()[Setting::max_query_size], max_depth, max_backtracks); settings.emplace(std::move(settings_ast->as()->changes)); } } @@ -198,16 +209,16 @@ void DDLTaskBase::parseQueryFromEntry(ContextPtr context) const char * end = begin + entry.query.size(); const auto & settings = context->getSettingsRef(); - ParserQuery parser_query(end, settings.allow_settings_after_format_in_insert); + ParserQuery parser_query(end, settings[Setting::allow_settings_after_format_in_insert]); String description; - query = parseQuery(parser_query, begin, end, description, 0, settings.max_parser_depth, settings.max_parser_backtracks); + query = parseQuery(parser_query, begin, end, description, 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } void DDLTaskBase::formatRewrittenQuery(ContextPtr context) { /// Convert rewritten AST back to string. query_str = queryToString(*query); - query_for_logging = query->formatForLogging(context->getSettingsRef().log_queries_cut_to_length); + query_for_logging = query->formatForLogging(context->getSettingsRef()[Setting::log_queries_cut_to_length]); } ContextMutablePtr DDLTaskBase::makeQueryContext(ContextPtr from_context, const ZooKeeperPtr & /*zookeeper*/) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 697fd0f406b..d922b4438b9 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -52,6 +52,12 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool implicit_transaction; + extern const SettingsUInt64 readonly; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; +} namespace ErrorCodes { @@ -115,7 +121,7 @@ DDLWorker::DDLWorker( context->setSetting("profile", config->getString(prefix + ".profile")); } - if (context->getSettingsRef().readonly) + if (context->getSettingsRef()[Setting::readonly]) { LOG_WARNING(log, "Distributed DDL worker is run with readonly settings, it will not be able to execute DDL queries Set appropriate system_profile or distributed_ddl.profile to fix this."); } @@ -483,9 +489,9 @@ bool DDLWorker::tryExecuteQuery(DDLTaskBase & task, const ZooKeeperPtr & zookeep auto query_context = task.makeQueryContext(context, zookeeper); chassert(!query_context->getCurrentTransaction()); - if (query_context->getSettingsRef().implicit_transaction) + if (query_context->getSettingsRef()[Setting::implicit_transaction]) { - if (query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (query_context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot begin an implicit transaction inside distributed DDL query"); query_context->setSetting("implicit_transaction", Field{0}); } diff --git a/src/Interpreters/DDLWorker.h b/src/Interpreters/DDLWorker.h index 6d1dabda54f..8af50bca69b 100644 --- a/src/Interpreters/DDLWorker.h +++ b/src/Interpreters/DDLWorker.h @@ -7,11 +7,13 @@ #include #include #include -#include +#include +#include #include #include #include +#include #include #include #include diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 1ca8c40460c..2597fe3592f 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -32,6 +32,12 @@ extern const Event ScalarSubqueriesCacheMiss; namespace DB { +namespace Setting +{ + extern const SettingsBool enable_scalar_subquery_optimization; + extern const SettingsBool extremes; + extern const SettingsUInt64 max_result_rows; +} namespace ErrorCodes { @@ -75,8 +81,8 @@ static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqu { auto subquery_context = Context::createCopy(data.getContext()); Settings subquery_settings = data.getContext()->getSettingsCopy(); - subquery_settings.max_result_rows = 1; - subquery_settings.extremes = false; + subquery_settings[Setting::max_result_rows] = 1; + subquery_settings[Setting::extremes] = false; subquery_context->setSettings(subquery_settings); if (subquery_context->hasQueryContext()) @@ -268,9 +274,7 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr const Settings & settings = data.getContext()->getSettingsRef(); // Always convert to literals when there is no query context. - if (data.only_analyze - || !settings.enable_scalar_subquery_optimization - || worthConvertingScalarToLiteral(scalar, data.max_literal_size) + if (data.only_analyze || !settings[Setting::enable_scalar_subquery_optimization] || worthConvertingScalarToLiteral(scalar, data.max_literal_size) || !data.getContext()->hasQueryContext()) { auto lit = std::make_unique((*scalar.safeGetByPosition(0).column)[0]); diff --git a/src/Interpreters/ExpressionActionsSettings.cpp b/src/Interpreters/ExpressionActionsSettings.cpp index 8aec7afef4d..42864175826 100644 --- a/src/Interpreters/ExpressionActionsSettings.cpp +++ b/src/Interpreters/ExpressionActionsSettings.cpp @@ -5,16 +5,24 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool compile_expressions; + extern const SettingsUInt64 max_temporary_columns; + extern const SettingsUInt64 max_temporary_non_const_columns; + extern const SettingsUInt64 min_count_to_compile_expression; + extern const SettingsShortCircuitFunctionEvaluation short_circuit_function_evaluation; +} ExpressionActionsSettings ExpressionActionsSettings::fromSettings(const Settings & from, CompileExpressions compile_expressions) { ExpressionActionsSettings settings; - settings.can_compile_expressions = from.compile_expressions; - settings.min_count_to_compile_expression = from.min_count_to_compile_expression; - settings.max_temporary_columns = from.max_temporary_columns; - settings.max_temporary_non_const_columns = from.max_temporary_non_const_columns; + settings.can_compile_expressions = from[Setting::compile_expressions]; + settings.min_count_to_compile_expression = from[Setting::min_count_to_compile_expression]; + settings.max_temporary_columns = from[Setting::max_temporary_columns]; + settings.max_temporary_non_const_columns = from[Setting::max_temporary_non_const_columns]; settings.compile_expressions = compile_expressions; - settings.short_circuit_function_evaluation = from.short_circuit_function_evaluation; + settings.short_circuit_function_evaluation = from[Setting::short_circuit_function_evaluation]; return settings; } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 1235ebb0871..70b734f4b9f 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -87,6 +87,27 @@ namespace DB using LogAST = DebugASTLog; /// set to true to enable logs +namespace Setting +{ + extern const SettingsBool compile_sort_description; + extern const SettingsUInt64 distributed_group_by_no_merge; + extern const SettingsBool enable_early_constant_folding; + extern const SettingsBool enable_positional_arguments; + extern const SettingsBool group_by_use_nulls; + extern const SettingsUInt64 max_bytes_in_set; + extern const SettingsUInt64 max_rows_in_set; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_count_to_compile_aggregate_expression; + extern const SettingsUInt64 min_count_to_compile_sort_description; + extern const SettingsOverflowMode set_overflow_mode; + extern const SettingsBool optimize_aggregation_in_order; + extern const SettingsBool optimize_read_in_order; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsBool query_plan_aggregation_in_order; + extern const SettingsBool query_plan_read_in_order; + extern const SettingsUInt64 use_index_for_in_with_subqueries_max_values; +} + namespace ErrorCodes { @@ -107,7 +128,7 @@ namespace /// predicates because some performance tests use ignore function as a non-optimize guard. bool allowEarlyConstantFolding(const ActionsDAG & actions, const Settings & settings) { - if (!settings.enable_early_constant_folding) + if (!settings[Setting::enable_early_constant_folding]) return false; for (const auto & node : actions.getNodes()) @@ -150,14 +171,13 @@ bool sanitizeBlock(Block & block, bool throw_if_cannot_create_column) ExpressionAnalyzerData::~ExpressionAnalyzerData() = default; ExpressionAnalyzer::ExtractedSettings::ExtractedSettings(const Settings & settings_) - : use_index_for_in_with_subqueries(settings_.use_index_for_in_with_subqueries) - , size_limits_for_set(settings_.max_rows_in_set, settings_.max_bytes_in_set, settings_.set_overflow_mode) + : size_limits_for_set(settings_[Setting::max_rows_in_set], settings_[Setting::max_bytes_in_set], settings_[Setting::set_overflow_mode]) , size_limits_for_set_used_with_index( - (settings_.use_index_for_in_with_subqueries_max_values && - settings_.use_index_for_in_with_subqueries_max_values < settings_.max_rows_in_set) ? - size_limits_for_set : - SizeLimits(settings_.use_index_for_in_with_subqueries_max_values, settings_.max_bytes_in_set, OverflowMode::BREAK)) - , distributed_group_by_no_merge(settings_.distributed_group_by_no_merge) + (settings_[Setting::use_index_for_in_with_subqueries_max_values] + && settings_[Setting::use_index_for_in_with_subqueries_max_values] < settings_[Setting::max_rows_in_set]) + ? size_limits_for_set + : SizeLimits(settings_[Setting::use_index_for_in_with_subqueries_max_values], settings_[Setting::max_bytes_in_set], OverflowMode::BREAK)) + , distributed_group_by_no_merge(settings_[Setting::distributed_group_by_no_merge]) {} ExpressionAnalyzer::~ExpressionAnalyzer() = default; @@ -293,7 +313,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) group_by_kind = GroupByKind::GROUPING_SETS; else group_by_kind = GroupByKind::ORDINARY; - bool use_nulls = group_by_kind != GroupByKind::ORDINARY && getContext()->getSettingsRef().group_by_use_nulls; + bool use_nulls = group_by_kind != GroupByKind::ORDINARY && getContext()->getSettingsRef()[Setting::group_by_use_nulls]; /// For GROUPING SETS with multiple groups we always add virtual __grouping_set column /// With set number, which is used as an additional key at the stage of merging aggregating data. @@ -304,7 +324,7 @@ void ExpressionAnalyzer::analyzeAggregation(ActionsDAG & temp_actions) { ssize_t size = group_asts.size(); - if (getContext()->getSettingsRef().enable_positional_arguments) + if (getContext()->getSettingsRef()[Setting::enable_positional_arguments]) replaceForPositionalArguments(group_asts[i], select_query, ASTSelectQuery::Expression::GROUP_BY); if (select_query->group_by_with_grouping_sets) @@ -853,8 +873,8 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAG & actions) } } - bool compile_sort_description = current_context->getSettingsRef().compile_sort_description; - size_t min_count_to_compile_sort_description = current_context->getSettingsRef().min_count_to_compile_sort_description; + bool compile_sort_description = current_context->getSettingsRef()[Setting::compile_sort_description]; + size_t min_count_to_compile_sort_description = current_context->getSettingsRef()[Setting::min_count_to_compile_sort_description]; for (auto & [_, window_description] : window_descriptions) { @@ -1006,7 +1026,7 @@ static std::shared_ptr tryCreateJoin( if (analyzed_join->allowParallelHashJoin()) return std::make_shared( - context, analyzed_join, settings.max_threads, right_sample_block, StatsCollectingParams{}); + context, analyzed_join, settings[Setting::max_threads], right_sample_block, StatsCollectingParams{}); return std::make_shared(analyzed_join, right_sample_block); } @@ -1540,8 +1560,8 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain, appendSelectSkipWindowExpressions(step, child); } -ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, - ManyExpressionActions & order_by_elements_actions) +ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy( + ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, ManyExpressionActions & order_by_elements_actions) { const auto * select_query = getSelectQuery(); @@ -1560,7 +1580,7 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(Expr if (!ast || ast->children.empty()) throw Exception(ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE, "Bad ORDER BY expression AST"); - if (getContext()->getSettingsRef().enable_positional_arguments) + if (getContext()->getSettingsRef()[Setting::enable_positional_arguments]) replaceForPositionalArguments(ast->children.at(0), select_query, ASTSelectQuery::Expression::ORDER_BY); } @@ -1672,7 +1692,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain auto & children = select_query->limitBy()->children; for (auto & child : children) { - if (getContext()->getSettingsRef().enable_positional_arguments) + if (getContext()->getSettingsRef()[Setting::enable_positional_arguments]) replaceForPositionalArguments(child, select_query, ASTSelectQuery::Expression::LIMIT_BY); auto child_name = child->getColumnName(); @@ -1912,7 +1932,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( { ExpressionActionsChain chain(context); - if (storage && (query.sampleSize() || settings.parallel_replicas_count > 1)) + if (storage && (query.sampleSize() || settings[Setting::parallel_replicas_count] > 1)) { // we evaluate sampling for Merge lazily, so we need to get all the columns if (storage->getName() == "Merge") @@ -2009,16 +2029,14 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( if (need_aggregate) { /// TODO correct conditions - optimize_aggregation_in_order = - context->getSettingsRef().optimize_aggregation_in_order - && (!context->getSettingsRef().query_plan_aggregation_in_order) - && storage && query.groupBy(); + optimize_aggregation_in_order = context->getSettingsRef()[Setting::optimize_aggregation_in_order] + && (!context->getSettingsRef()[Setting::query_plan_aggregation_in_order]) && storage && query.groupBy(); query_analyzer.appendGroupBy(chain, only_types || !first_stage, optimize_aggregation_in_order, group_by_elements_actions); query_analyzer.appendAggregateFunctionsArguments(chain, only_types || !first_stage); before_aggregation = chain.getLastActions(); - if (settings.group_by_use_nulls) + if (settings[Setting::group_by_use_nulls]) query_analyzer.appendGroupByModifiers(before_aggregation->dag, chain, only_types); auto columns_before_aggregation = finalize_chain(chain); @@ -2050,8 +2068,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( bool has_grouping = query_analyzer.group_by_kind != GroupByKind::ORDINARY; auto actual_header = Aggregator::Params::getHeader( header_before_aggregation, /*only_merge*/ false, keys, aggregates, /*final*/ true); - actual_header = AggregatingStep::appendGroupingColumn( - std::move(actual_header), keys, has_grouping, settings.group_by_use_nulls); + actual_header + = AggregatingStep::appendGroupingColumn(std::move(actual_header), keys, has_grouping, settings[Setting::group_by_use_nulls]); Block expected_header; for (const auto & expected : query_analyzer.aggregated_columns) @@ -2087,14 +2105,8 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( join_allow_read_in_order = typeid_cast(join.get()) && !join_has_delayed_stream; } - optimize_read_in_order = - settings.optimize_read_in_order && (!settings.query_plan_read_in_order) - && storage - && query.orderBy() - && !query_analyzer.hasAggregation() - && !query_analyzer.hasWindow() - && !query.final() - && join_allow_read_in_order; + optimize_read_in_order = settings[Setting::optimize_read_in_order] && (!settings[Setting::query_plan_read_in_order]) && storage && query.orderBy() + && !query_analyzer.hasAggregation() && !query_analyzer.hasWindow() && !query.final() && join_allow_read_in_order; /// If there is aggregation, we execute expressions in SELECT and ORDER BY on the initiating server, otherwise on the source servers. query_analyzer.appendSelect(chain, only_types || (need_aggregate ? !second_stage : !first_stage)); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 483be5cc4c4..d4ee8832c1c 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -90,7 +90,6 @@ private: /// Extracts settings to enlight which are used (and avoid copy of others). struct ExtractedSettings { - const bool use_index_for_in_with_subqueries; const SizeLimits size_limits_for_set; const SizeLimits size_limits_for_set_used_with_index; const UInt64 distributed_group_by_no_merge; diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 5f6d7e33927..bfd581315e4 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -16,6 +16,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -80,7 +84,7 @@ ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(co std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); auto dictionary = std::static_pointer_cast(load(resolved_dictionary_name)); - if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries) + if (local_context->hasQueryContext() && local_context->getSettingsRef()[Setting::log_queries]) local_context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, dictionary->getQualifiedName()); return dictionary; @@ -91,7 +95,7 @@ ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::tryGetDictionary std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); auto dictionary = std::static_pointer_cast(tryLoad(resolved_dictionary_name)); - if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries && dictionary) + if (local_context->hasQueryContext() && local_context->getSettingsRef()[Setting::log_queries] && dictionary) local_context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, dictionary->getQualifiedName()); return dictionary; diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index b1745ea8956..a821a8dcfc2 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -28,6 +28,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsBool parallel_replicas_allow_in_with_subquery; + extern const SettingsBool prefer_global_in_and_join; +} namespace ErrorCodes { @@ -206,7 +212,7 @@ private: static void visit(ASTFunction & func, ASTPtr &, Data & data) { const Settings & settings = data.getContext()->getSettingsRef(); - const bool prefer_global = settings.prefer_global_in_and_join; + const bool prefer_global = settings[Setting::prefer_global_in_and_join]; const bool enable_parallel_processing_of_joins = data.getContext()->canUseParallelReplicasOnInitiator(); if (((prefer_global || enable_parallel_processing_of_joins) @@ -217,15 +223,15 @@ private: if (enable_parallel_processing_of_joins) { /// We don't enable parallel replicas for IN (subquery) - if (!settings.parallel_replicas_allow_in_with_subquery && ast->as()) + if (!settings[Setting::parallel_replicas_allow_in_with_subquery] && ast->as()) { - if (settings.allow_experimental_parallel_reading_from_replicas == 1) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] == 1) { LOG_DEBUG(getLogger("GlobalSubqueriesMatcher"), "IN with subquery is not supported with parallel replicas"); data.getContext()->getQueryContext()->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); return; } - else if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + else if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas"); } } @@ -254,7 +260,7 @@ private: static void visit(ASTTablesInSelectQueryElement & table_elem, ASTPtr &, Data & data) { const Settings & settings = data.getContext()->getSettingsRef(); - const bool prefer_global = settings.prefer_global_in_and_join; + const bool prefer_global = settings[Setting::prefer_global_in_and_join]; const bool enable_parallel_processing_of_joins = data.getContext()->canUseParallelReplicasOnInitiator(); if (table_elem.table_join @@ -277,13 +283,13 @@ private: if (!is_subquery) { - if (settings.allow_experimental_parallel_reading_from_replicas == 1) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] == 1) { LOG_DEBUG(getLogger("GlobalSubqueriesMatcher"), "JOIN with parallel replicas is only supported with subqueries"); data.getContext()->getQueryContext()->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); return; } - else if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + else if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JOIN with parallel replicas is only supported with subqueries"); } } diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp index a241d1cd258..978782c851f 100644 --- a/src/Interpreters/GraceHashJoin.cpp +++ b/src/Interpreters/GraceHashJoin.cpp @@ -23,6 +23,11 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 grace_hash_join_initial_buckets; + extern const SettingsUInt64 grace_hash_join_max_buckets; +} namespace ErrorCodes { @@ -254,7 +259,8 @@ void flushBlocksToBuckets(Blocks & blocks, const GraceHashJoin::Buckets & bucket } GraceHashJoin::GraceHashJoin( - ContextPtr context_, std::shared_ptr table_join_, + ContextPtr context_, + std::shared_ptr table_join_, const Block & left_sample_block_, const Block & right_sample_block_, TemporaryDataOnDiskScopePtr tmp_data_, @@ -265,8 +271,7 @@ GraceHashJoin::GraceHashJoin( , left_sample_block{left_sample_block_} , right_sample_block{right_sample_block_} , any_take_last_row{any_take_last_row_} - , max_num_buckets{context->getSettingsRef().grace_hash_join_max_buckets} - , max_block_size{context->getSettingsRef().max_block_size} + , max_num_buckets{context->getSettingsRef()[Setting::grace_hash_join_max_buckets]} , left_key_names(table_join->getOnlyClause().key_names_left) , right_key_names(table_join->getOnlyClause().key_names_right) , tmp_data(std::make_unique(tmp_data_, CurrentMetrics::TemporaryFilesForJoin)) @@ -284,7 +289,8 @@ void GraceHashJoin::initBuckets() const auto & settings = context->getSettingsRef(); - size_t initial_num_buckets = roundUpToPowerOfTwoOrZero(std::clamp(settings.grace_hash_join_initial_buckets, 1, settings.grace_hash_join_max_buckets)); + size_t initial_num_buckets = roundUpToPowerOfTwoOrZero( + std::clamp(settings[Setting::grace_hash_join_initial_buckets], 1, settings[Setting::grace_hash_join_max_buckets])); addBuckets(initial_num_buckets); diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h index ff396683230..d31d6886af7 100644 --- a/src/Interpreters/GraceHashJoin.h +++ b/src/Interpreters/GraceHashJoin.h @@ -128,7 +128,6 @@ private: Block output_sample_block; bool any_take_last_row; const size_t max_num_buckets; - size_t max_block_size; Names left_key_names; Names right_key_names; diff --git a/src/Interpreters/IInterpreter.cpp b/src/Interpreters/IInterpreter.cpp index a291199f24b..c4255b8a290 100644 --- a/src/Interpreters/IInterpreter.cpp +++ b/src/Interpreters/IInterpreter.cpp @@ -6,6 +6,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool throw_on_unsupported_query_inside_transaction; +} namespace ErrorCodes { @@ -39,7 +43,7 @@ void IInterpreter::checkStorageSupportsTransactionsIfNeeded(const StoragePtr & s if (storage->supportsTransactions()) return; - if (context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Storage {} (table {}) does not support transactions", storage->getName(), storage->getStorageID().getNameForLogs()); diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp index 0034b50f02c..1bb60a4cf96 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp @@ -16,13 +16,32 @@ namespace DB { +namespace Setting +{ + extern const SettingsString additional_result_filter; + extern const SettingsUInt64 max_bytes_to_read; + extern const SettingsUInt64 max_bytes_to_read_leaf; + extern const SettingsSeconds max_estimated_execution_time; + extern const SettingsUInt64 max_execution_speed; + extern const SettingsUInt64 max_execution_speed_bytes; + extern const SettingsSeconds max_execution_time; + extern const SettingsUInt64 max_query_size; + extern const SettingsUInt64 max_rows_to_read; + extern const SettingsUInt64 max_rows_to_read_leaf; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_execution_speed; + extern const SettingsUInt64 min_execution_speed_bytes; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsOverflowMode read_overflow_mode; + extern const SettingsOverflowMode read_overflow_mode_leaf; + extern const SettingsSeconds timeout_before_checking_execution_speed; + extern const SettingsOverflowMode timeout_overflow_mode; +} -IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_, - const ContextMutablePtr & context_, const SelectQueryOptions & options_) - : query_ptr(query_ptr_) - , context(context_) - , options(options_) - , max_streams(context->getSettingsRef().max_threads) +IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery( + const ASTPtr & query_ptr_, const ContextMutablePtr & context_, const SelectQueryOptions & options_) + : query_ptr(query_ptr_), context(context_), options(options_), max_streams(context->getSettingsRef()[Setting::max_threads]) { /// FIXME All code here will work with the old analyzer, however for views over Distributed tables /// it's possible that new analyzer will be enabled in ::getQueryProcessingStage method @@ -57,9 +76,9 @@ static StreamLocalLimits getLimitsForStorage(const Settings & settings, const Se { StreamLocalLimits limits; limits.mode = LimitsMode::LIMITS_TOTAL; - limits.size_limits = SizeLimits(settings.max_rows_to_read, settings.max_bytes_to_read, settings.read_overflow_mode); - limits.speed_limits.max_execution_time = settings.max_execution_time; - limits.timeout_overflow_mode = settings.timeout_overflow_mode; + limits.size_limits = SizeLimits(settings[Setting::max_rows_to_read], settings[Setting::max_bytes_to_read], settings[Setting::read_overflow_mode]); + limits.speed_limits.max_execution_time = settings[Setting::max_execution_time]; + limits.timeout_overflow_mode = settings[Setting::timeout_overflow_mode]; /** Quota and minimal speed restrictions are checked on the initiating server of the request, and not on remote servers, * because the initiating server has a summary of the execution of the request on all servers. @@ -72,14 +91,14 @@ static StreamLocalLimits getLimitsForStorage(const Settings & settings, const Se */ if (options.to_stage == QueryProcessingStage::Complete) { - limits.speed_limits.min_execution_rps = settings.min_execution_speed; - limits.speed_limits.min_execution_bps = settings.min_execution_speed_bytes; + limits.speed_limits.min_execution_rps = settings[Setting::min_execution_speed]; + limits.speed_limits.min_execution_bps = settings[Setting::min_execution_speed_bytes]; } - limits.speed_limits.max_execution_rps = settings.max_execution_speed; - limits.speed_limits.max_execution_bps = settings.max_execution_speed_bytes; - limits.speed_limits.timeout_before_checking_execution_speed = settings.timeout_before_checking_execution_speed; - limits.speed_limits.max_estimated_execution_time = settings.max_estimated_execution_time; + limits.speed_limits.max_execution_rps = settings[Setting::max_execution_speed]; + limits.speed_limits.max_execution_bps = settings[Setting::max_execution_speed_bytes]; + limits.speed_limits.timeout_before_checking_execution_speed = settings[Setting::timeout_before_checking_execution_speed]; + limits.speed_limits.max_estimated_execution_time = settings[Setting::max_estimated_execution_time]; return limits; } @@ -95,7 +114,7 @@ StorageLimits IInterpreterUnionOrSelectQuery::getStorageLimits(const Context & c if (!options.ignore_limits) { limits = getLimitsForStorage(settings, options); - leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, settings.max_bytes_to_read_leaf, settings.read_overflow_mode_leaf); + leaf_limits = SizeLimits(settings[Setting::max_rows_to_read_leaf], settings[Setting::max_bytes_to_read_leaf], settings[Setting::read_overflow_mode_leaf]); } return {limits, leaf_limits}; @@ -114,14 +133,19 @@ void IInterpreterUnionOrSelectQuery::setQuota(QueryPipeline & pipeline) const static ASTPtr parseAdditionalPostFilter(const Context & context) { const auto & settings = context.getSettingsRef(); - const String & filter = settings.additional_result_filter; + const String & filter = settings[Setting::additional_result_filter]; if (filter.empty()) return nullptr; ParserExpression parser; return parseQuery( - parser, filter.data(), filter.data() + filter.size(), - "additional filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parser, + filter.data(), + filter.data() + filter.size(), + "additional filter", + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); } static ActionsDAG makeAdditionalPostFilter(ASTPtr & ast, ContextPtr context, const Block & header) diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 24e594661e9..ae5e70d1dcd 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool asterisk_include_alias_columns; + extern const SettingsBool asterisk_include_materialized_columns; +} namespace ErrorCodes { @@ -314,9 +319,8 @@ IdentifierMembershipCollector::IdentifierMembershipCollector(const ASTSelectQuer QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select()); const auto & settings = context->getSettingsRef(); - tables = getDatabaseAndTablesWithColumns(getTableExpressions(select), context, - settings.asterisk_include_alias_columns, - settings.asterisk_include_materialized_columns); + tables = getDatabaseAndTablesWithColumns( + getTableExpressions(select), context, settings[Setting::asterisk_include_alias_columns], settings[Setting::asterisk_include_materialized_columns]); } std::optional IdentifierMembershipCollector::getIdentsMembership(ASTPtr ast) const diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 22ecb6a5db2..02b1daf901c 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -16,6 +16,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsDistributedProductMode distributed_product_mode; + extern const SettingsBool prefer_global_in_and_join; +} namespace ErrorCodes { @@ -67,7 +72,7 @@ struct NonGlobalTableData : public WithContext private: void renameIfNeeded(ASTPtr & database_and_table) { - const DistributedProductMode distributed_product_mode = getContext()->getSettingsRef().distributed_product_mode; + const DistributedProductMode distributed_product_mode = getContext()->getSettingsRef()[Setting::distributed_product_mode]; StoragePtr storage = tryGetTable(database_and_table, getContext()); if (!storage || !checker.hasAtLeastTwoShards(*storage)) @@ -90,7 +95,7 @@ private: renamed_tables.emplace_back(identifier.clone()); identifier.resetTable(database, table); } - else if (getContext()->getSettingsRef().prefer_global_in_and_join || distributed_product_mode == DistributedProductMode::GLOBAL) + else if (getContext()->getSettingsRef()[Setting::prefer_global_in_and_join] || distributed_product_mode == DistributedProductMode::GLOBAL) { if (function) { @@ -234,7 +239,7 @@ void InJoinSubqueriesPreprocessor::visit(ASTPtr & ast) const if (!query || !query->tables()) return; - if (getContext()->getSettingsRef().distributed_product_mode == DistributedProductMode::ALLOW) + if (getContext()->getSettingsRef()[Setting::distributed_product_mode] == DistributedProductMode::ALLOW) return; const auto & tables_in_select_query = query->tables()->as(); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 9b5b5dfc20a..e0b17213869 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -38,6 +38,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_statistics; + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -129,7 +134,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) checkStorageSupportsTransactionsIfNeeded(table, getContext()); if (table->isStaticStorage()) throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only"); - auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); if (modify_query) { @@ -178,7 +183,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) else throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong parameter type in ALTER query"); - if (!getContext()->getSettingsRef().allow_experimental_statistics + if (!getContext()->getSettingsRef()[Setting::allow_experimental_statistics] && (command_ast->type == ASTAlterCommand::ADD_STATISTICS || command_ast->type == ASTAlterCommand::DROP_STATISTICS || command_ast->type == ASTAlterCommand::MATERIALIZE_STATISTICS)) throw Exception(ErrorCodes::INCORRECT_QUERY, "Alter table with statistics is now disabled. Turn on allow_experimental_statistics"); @@ -201,7 +206,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (!alter_commands.empty()) { - auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout); + auto alter_lock = table->lockForAlter(getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); alter_commands.validate(table, getContext()); alter_commands.prepare(metadata); diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 2b022a8cee0..72a099fb455 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -36,6 +36,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool check_query_single_value_result; + extern const SettingsMaxThreads max_threads; +} namespace ErrorCodes { @@ -422,7 +427,7 @@ BlockIO InterpreterCheckQuery::execute() throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected query {} in InterpreterCheckQuery", query_ptr->formatForErrorMessage()); } - size_t num_streams = std::max(settings.max_threads, 1); + size_t num_streams = std::max(settings[Setting::max_threads], 1); processors->emplace_back(worker_source); std::vector worker_ports; @@ -463,7 +468,7 @@ BlockIO InterpreterCheckQuery::execute() resize_outport = &resize_processor->getOutputs().front(); } - if (settings.check_query_single_value_result) + if (settings[Setting::check_query_single_value_result]) { chassert(!processors->empty() && !processors->back()->getOutputs().empty()); Block header = processors->back()->getOutputs().front().getHeader(); diff --git a/src/Interpreters/InterpreterCreateIndexQuery.cpp b/src/Interpreters/InterpreterCreateIndexQuery.cpp index 1c6abe842d4..cb474ecdc3c 100644 --- a/src/Interpreters/InterpreterCreateIndexQuery.cpp +++ b/src/Interpreters/InterpreterCreateIndexQuery.cpp @@ -15,6 +15,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_create_index_without_type; + extern const SettingsBool create_index_ignore_unique; + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -32,7 +38,7 @@ BlockIO InterpreterCreateIndexQuery::execute() if (create_index.unique) { - if (!current_context->getSettingsRef().create_index_ignore_unique) + if (!current_context->getSettingsRef()[Setting::create_index_ignore_unique]) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "CREATE UNIQUE INDEX is not supported." " SET create_index_ignore_unique=1 to ignore this UNIQUE keyword."); @@ -42,7 +48,7 @@ BlockIO InterpreterCreateIndexQuery::execute() // Noop if allow_create_index_without_type = true. throw otherwise if (!create_index.index_decl->as()->getType()) { - if (!current_context->getSettingsRef().allow_create_index_without_type) + if (!current_context->getSettingsRef()[Setting::allow_create_index_without_type]) { throw Exception(ErrorCodes::INCORRECT_QUERY, "CREATE INDEX without TYPE is forbidden." " SET allow_create_index_without_type=1 to ignore this statements"); @@ -92,7 +98,7 @@ BlockIO InterpreterCreateIndexQuery::execute() alter_commands.emplace_back(std::move(command)); - auto alter_lock = table->lockForAlter(current_context->getSettingsRef().lock_acquire_timeout); + auto alter_lock = table->lockForAlter(current_context->getSettingsRef()[Setting::lock_acquire_timeout]); StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); alter_commands.validate(table, current_context); alter_commands.prepare(metadata); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 60d4abd0ef8..e161c0455b9 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -96,6 +96,45 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_codecs; + extern const SettingsBool allow_experimental_database_materialized_mysql; + extern const SettingsBool allow_experimental_database_materialized_postgresql; + extern const SettingsBool allow_experimental_full_text_index; + extern const SettingsBool allow_experimental_inverted_index; + extern const SettingsBool allow_experimental_refreshable_materialized_view; + extern const SettingsBool allow_experimental_statistics; + extern const SettingsBool allow_experimental_vector_similarity_index; + extern const SettingsBool allow_materialized_view_with_bad_select; + extern const SettingsBool allow_suspicious_codecs; + extern const SettingsBool compatibility_ignore_collation_in_create_table; + extern const SettingsBool compatibility_ignore_auto_increment_in_create_table; + extern const SettingsBool create_if_not_exists; + extern const SettingsFloat create_replicated_merge_tree_fault_injection_probability; + extern const SettingsBool database_atomic_wait_for_drop_and_detach_synchronously; + extern const SettingsUInt64 database_replicated_allow_explicit_uuid; + extern const SettingsBool database_replicated_allow_heavy_create; + extern const SettingsBool database_replicated_allow_only_replicated_engine; + extern const SettingsBool data_type_default_nullable; + extern const SettingsSQLSecurityType default_materialized_view_sql_security; + extern const SettingsSQLSecurityType default_normal_view_sql_security; + extern const SettingsDefaultTableEngine default_table_engine; + extern const SettingsDefaultTableEngine default_temporary_table_engine; + extern const SettingsString default_view_definer; + extern const SettingsUInt64 distributed_ddl_entry_format_version; + extern const SettingsBool enable_deflate_qpl_codec; + extern const SettingsBool enable_zstd_qat_codec; + extern const SettingsBool flatten_nested; + extern const SettingsBool fsync_metadata; + extern const SettingsBool insert_allow_materialized_columns; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsBool restore_replace_external_engines_to_null; + extern const SettingsBool restore_replace_external_table_functions_to_null; +} namespace ErrorCodes { @@ -285,8 +324,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) } if ((create.storage->engine->name == "MaterializeMySQL" || create.storage->engine->name == "MaterializedMySQL") - && !getContext()->getSettingsRef().allow_experimental_database_materialized_mysql - && !internal && !create.attach) + && !getContext()->getSettingsRef()[Setting::allow_experimental_database_materialized_mysql] && !internal && !create.attach) { throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "MaterializedMySQL is an experimental database engine. " @@ -294,8 +332,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) } if (create.storage->engine->name == "MaterializedPostgreSQL" - && !getContext()->getSettingsRef().allow_experimental_database_materialized_postgresql - && !internal && !create.attach) + && !getContext()->getSettingsRef()[Setting::allow_experimental_database_materialized_postgresql] && !internal && !create.attach) { throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "MaterializedPostgreSQL is an experimental database engine. " @@ -337,7 +374,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) writeString(statement, out); out.next(); - if (getContext()->getSettingsRef().fsync_metadata) + if (getContext()->getSettingsRef()[Setting::fsync_metadata]) out.sync(); out.close(); } @@ -543,14 +580,15 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( ASTPtr default_expr_list = std::make_shared(); NamesAndTypesList column_names_and_types; - bool make_columns_nullable = mode <= LoadingStrictnessLevel::SECONDARY_CREATE && !is_restore_from_backup && context_->getSettingsRef().data_type_default_nullable; + bool make_columns_nullable = mode <= LoadingStrictnessLevel::SECONDARY_CREATE && !is_restore_from_backup + && context_->getSettingsRef()[Setting::data_type_default_nullable]; bool has_columns_with_default_without_type = false; for (const auto & ast : columns_ast.children) { const auto & col_decl = ast->as(); - if (col_decl.collation && !context_->getSettingsRef().compatibility_ignore_collation_in_create_table) + if (col_decl.collation && !context_->getSettingsRef()[Setting::compatibility_ignore_collation_in_create_table]) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot support collation, please set compatibility_ignore_collation_in_create_table=true"); } @@ -637,10 +675,10 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( } bool skip_checks = LoadingStrictnessLevel::SECONDARY_CREATE <= mode; - bool sanity_check_compression_codecs = !skip_checks && !context_->getSettingsRef().allow_suspicious_codecs; - bool allow_experimental_codecs = skip_checks || context_->getSettingsRef().allow_experimental_codecs; - bool enable_deflate_qpl_codec = skip_checks || context_->getSettingsRef().enable_deflate_qpl_codec; - bool enable_zstd_qat_codec = skip_checks || context_->getSettingsRef().enable_zstd_qat_codec; + bool sanity_check_compression_codecs = !skip_checks && !context_->getSettingsRef()[Setting::allow_suspicious_codecs]; + bool allow_experimental_codecs = skip_checks || context_->getSettingsRef()[Setting::allow_experimental_codecs]; + bool enable_deflate_qpl_codec = skip_checks || context_->getSettingsRef()[Setting::enable_deflate_qpl_codec]; + bool enable_zstd_qat_codec = skip_checks || context_->getSettingsRef()[Setting::enable_zstd_qat_codec]; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -654,7 +692,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( /// ignore or not other database extensions depending on compatibility settings if (col_decl.default_specifier == "AUTO_INCREMENT" - && !context_->getSettingsRef().compatibility_ignore_auto_increment_in_create_table) + && !context_->getSettingsRef()[Setting::compatibility_ignore_auto_increment_in_create_table]) { throw Exception(ErrorCodes::SYNTAX_ERROR, "AUTO_INCREMENT is not supported. To ignore the keyword " @@ -706,8 +744,9 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.statistics_desc) { - if (!skip_checks && !context_->getSettingsRef().allow_experimental_statistics) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Create table with statistics is now disabled. Turn on allow_experimental_statistics"); + if (!skip_checks && !context_->getSettingsRef()[Setting::allow_experimental_statistics]) + throw Exception( + ErrorCodes::INCORRECT_QUERY, "Create table with statistics is now disabled. Turn on allow_experimental_statistics"); column.statistics = ColumnStatisticsDescription::fromColumnDeclaration(col_decl, column.type); } @@ -723,7 +762,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( res.add(std::move(column)); } - if (mode <= LoadingStrictnessLevel::SECONDARY_CREATE && !is_restore_from_backup && context_->getSettingsRef().flatten_nested) + if (mode <= LoadingStrictnessLevel::SECONDARY_CREATE && !is_restore_from_backup && context_->getSettingsRef()[Setting::flatten_nested]) res.flattenNested(); @@ -783,14 +822,14 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti throw Exception(ErrorCodes::ILLEGAL_INDEX, "Duplicated index name {} is not allowed. Please use different index names.", backQuoteIfNeed(index_desc.name)); const auto & settings = getContext()->getSettingsRef(); - if (index_desc.type == FULL_TEXT_INDEX_NAME && !settings.allow_experimental_full_text_index) + if (index_desc.type == FULL_TEXT_INDEX_NAME && !settings[Setting::allow_experimental_full_text_index]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental full-text index feature is disabled. Turn on setting 'allow_experimental_full_text_index'"); /// ---- /// Temporary check during a transition period. Please remove at the end of 2024. - if (index_desc.type == INVERTED_INDEX_NAME && !settings.allow_experimental_inverted_index) + if (index_desc.type == INVERTED_INDEX_NAME && !settings[Setting::allow_experimental_inverted_index]) throw Exception(ErrorCodes::ILLEGAL_INDEX, "Please use index type 'full_text' instead of 'inverted'"); /// ---- - if (index_desc.type == "vector_similarity" && !settings.allow_experimental_vector_similarity_index) + if (index_desc.type == "vector_similarity" && !settings[Setting::allow_experimental_vector_similarity_index]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental vector similarity index is disabled. Turn on setting 'allow_experimental_vector_similarity_index'"); properties.indices.push_back(index_desc); @@ -811,7 +850,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, getContext()); /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. - as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); auto as_storage_metadata = as_storage->getInMemoryMetadataPtr(); properties.columns = as_storage_metadata->getColumns(); @@ -840,7 +879,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti Block as_select_sample; - if (getContext()->getSettingsRef().allow_experimental_analyzer) + if (getContext()->getSettingsRef()[Setting::allow_experimental_analyzer]) { as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); } @@ -981,7 +1020,7 @@ void InterpreterCreateQuery::validateMaterializedViewColumnsAndEngine(const ASTC { try { - if (getContext()->getSettingsRef().allow_experimental_analyzer) + if (getContext()->getSettingsRef()[Setting::allow_experimental_analyzer]) { input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext()); } @@ -994,7 +1033,7 @@ void InterpreterCreateQuery::validateMaterializedViewColumnsAndEngine(const ASTC } catch (Exception &) { - if (!getContext()->getSettingsRef().allow_materialized_view_with_bad_select) + if (!getContext()->getSettingsRef()[Setting::allow_materialized_view_with_bad_select]) throw; check_columns = false; } @@ -1082,7 +1121,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { if (create.as_table_function) { - if (getContext()->getSettingsRef().restore_replace_external_table_functions_to_null) + if (getContext()->getSettingsRef()[Setting::restore_replace_external_table_functions_to_null]) { const auto & factory = TableFunctionFactory::instance(); @@ -1119,7 +1158,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const } if (!create.storage->engine) - setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_temporary_table_engine.value); + setDefaultTableEngine(*create.storage, getContext()->getSettingsRef()[Setting::default_temporary_table_engine].value); checkTemporaryTableEngineName(create.storage->engine->name); return; @@ -1137,7 +1176,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (!to_engine->engine) { /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one. - setDefaultTableEngine(*to_engine, getContext()->getSettingsRef().default_table_engine.value); + setDefaultTableEngine(*to_engine, getContext()->getSettingsRef()[Setting::default_table_engine].value); } return; } @@ -1149,11 +1188,11 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (!create.storage->engine) { /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one. - setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value); + setDefaultTableEngine(*create.storage, getContext()->getSettingsRef()[Setting::default_table_engine].value); } /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to /// Null table engine. - else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null) + else if (getContext()->getSettingsRef()[Setting::restore_replace_external_engines_to_null]) { if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE) { @@ -1223,7 +1262,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { /// Set ENGINE by default. storage_def = std::make_shared(); - setDefaultTableEngine(*storage_def, getContext()->getSettingsRef().default_table_engine.value); + setDefaultTableEngine(*storage_def, getContext()->getSettingsRef()[Setting::default_table_engine].value); } /// Use the found table engine to modify the create query. @@ -1243,16 +1282,16 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data if (database->getEngineName() == "Replicated" && create.uuid != UUIDHelpers::Nil && !is_replicated_database_internal && !is_on_cluster && !create.attach) { - if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 0) + if (getContext()->getSettingsRef()[Setting::database_replicated_allow_explicit_uuid] == 0) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "It's not allowed to explicitly specify UUIDs for tables in Replicated databases, " "see database_replicated_allow_explicit_uuid"); } - else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 1) + else if (getContext()->getSettingsRef()[Setting::database_replicated_allow_explicit_uuid] == 1) { LOG_WARNING(&Poco::Logger::get("InterpreterCreateQuery"), "It's not recommended to explicitly specify UUIDs for tables in Replicated databases"); } - else if (getContext()->getSettingsRef().database_replicated_allow_explicit_uuid == 2) + else if (getContext()->getSettingsRef()[Setting::database_replicated_allow_explicit_uuid] == 2) { UUID old_uuid = create.uuid; create.uuid = UUIDHelpers::Nil; @@ -1467,7 +1506,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.refresh_strategy) { - if (!getContext()->getSettingsRef().allow_experimental_refreshable_materialized_view) + if (!getContext()->getSettingsRef()[Setting::allow_experimental_refreshable_materialized_view]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Refreshable materialized views are experimental. Enable allow_experimental_refreshable_materialized_view to use."); @@ -1498,7 +1537,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.select && create.is_materialized_view && mode <= LoadingStrictnessLevel::CREATE) validateMaterializedViewColumnsAndEngine(create, properties, database); - bool allow_heavy_populate = getContext()->getSettingsRef().database_replicated_allow_heavy_create && create.is_populate; + bool allow_heavy_populate = getContext()->getSettingsRef()[Setting::database_replicated_allow_heavy_create] && create.is_populate; if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate)) { bool is_storage_replicated = false; @@ -1696,7 +1735,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, /// old instance of the storage. For example, AsynchronousMetrics may cause ATTACH to fail, /// so we allow waiting here. If database_atomic_wait_for_drop_and_detach_synchronously is disabled /// and old storage instance still exists it will throw exception. - if (getContext()->getSettingsRef().database_atomic_wait_for_drop_and_detach_synchronously) + if (getContext()->getSettingsRef()[Setting::database_atomic_wait_for_drop_and_detach_synchronously]) database->waitDetachedTableNotInUse(create.uuid); else database->checkDetachedTableNotInUse(create.uuid); @@ -1754,7 +1793,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, res->getName()); } - if (!create.attach && getContext()->getSettingsRef().database_replicated_allow_only_replicated_engine) + if (!create.attach && getContext()->getSettingsRef()[Setting::database_replicated_allow_only_replicated_engine]) { bool is_replicated_storage = typeid_cast(res.get()) != nullptr; if (!is_replicated_storage && res->storesDataOnDisk() && database && database->getEngineName() == "Replicated") @@ -1771,7 +1810,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, auto * replicated_storage = typeid_cast(res.get()); if (replicated_storage) { - const auto probability = getContext()->getSettingsRef().create_replicated_merge_tree_fault_injection_probability; + const auto probability = getContext()->getSettingsRef()[Setting::create_replicated_merge_tree_fault_injection_probability]; std::bernoulli_distribution fault(probability); if (fault(thread_local_rng)) { @@ -1961,12 +2000,13 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create) insert->select = create.select->clone(); return InterpreterInsertQuery( - insert, - getContext(), - getContext()->getSettingsRef().insert_allow_materialized_columns, - /* no_squash */ false, - /* no_destination */ false, - /* async_isnert */ false).execute(); + insert, + getContext(), + getContext()->getSettingsRef()[Setting::insert_allow_materialized_columns], + /* no_squash */ false, + /* no_destination */ false, + /* async_isnert */ false) + .execute(); } return {}; @@ -1987,7 +2027,7 @@ void InterpreterCreateQuery::prepareOnClusterQuery(ASTCreateQuery & create, Cont if (cluster->maybeCrossReplication()) { - auto on_cluster_version = local_context->getSettingsRef().distributed_ddl_entry_format_version; + auto on_cluster_version = local_context->getSettingsRef()[Setting::distributed_ddl_entry_format_version]; if (DDLLogEntry::NORMALIZE_CREATE_ON_INITIATOR_VERSION <= on_cluster_version) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Value {} of setting distributed_ddl_entry_format_version " "is incompatible with cross-replication", on_cluster_version); @@ -2038,12 +2078,12 @@ BlockIO InterpreterCreateQuery::execute() FunctionNameNormalizer::visit(query_ptr.get()); auto & create = query_ptr->as(); - create.if_not_exists |= getContext()->getSettingsRef().create_if_not_exists; + create.if_not_exists |= getContext()->getSettingsRef()[Setting::create_if_not_exists]; bool is_create_database = create.database && !create.table; if (!create.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { - auto on_cluster_version = getContext()->getSettingsRef().distributed_ddl_entry_format_version; + auto on_cluster_version = getContext()->getSettingsRef()[Setting::distributed_ddl_entry_format_version]; if (is_create_database || on_cluster_version < DDLLogEntry::NORMALIZE_CREATE_ON_INITIATOR_VERSION) return executeQueryOnCluster(create); } @@ -2135,14 +2175,9 @@ void InterpreterCreateQuery::addColumnsDescriptionToCreateQueryIfNecessary(ASTCr return; auto ast_storage = std::make_shared(); - unsigned max_parser_depth = static_cast(getContext()->getSettingsRef().max_parser_depth); - unsigned max_parser_backtracks = static_cast(getContext()->getSettingsRef().max_parser_backtracks); - auto query_from_storage = DB::getCreateQueryFromStorage(storage, - ast_storage, - false, - max_parser_depth, - max_parser_backtracks, - true); + unsigned max_parser_depth_v = static_cast(getContext()->getSettingsRef()[Setting::max_parser_depth]); + unsigned max_parser_backtracks_v = static_cast(getContext()->getSettingsRef()[Setting::max_parser_backtracks]); + auto query_from_storage = DB::getCreateQueryFromStorage(storage, ast_storage, false, max_parser_depth_v, max_parser_backtracks_v, true); auto & create_query_from_storage = query_from_storage->as(); if (!create.columns_list) @@ -2165,13 +2200,13 @@ void InterpreterCreateQuery::processSQLSecurityOption(ContextPtr context_, ASTSQ SQLSecurityType default_security; if (is_materialized_view) - default_security = context_->getSettingsRef().default_materialized_view_sql_security; + default_security = context_->getSettingsRef()[Setting::default_materialized_view_sql_security]; else - default_security = context_->getSettingsRef().default_normal_view_sql_security; + default_security = context_->getSettingsRef()[Setting::default_normal_view_sql_security]; if (default_security == SQLSecurityType::DEFINER) { - String default_definer = context_->getSettingsRef().default_view_definer; + String default_definer = context_->getSettingsRef()[Setting::default_view_definer]; if (default_definer == "CURRENT_USER") sql_security.is_definer_current_user = true; else diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 5cbfdad4ff9..e732bba51e7 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -22,6 +22,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_lightweight_delete; + extern const SettingsUInt64 lightweight_deletes_sync; + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -64,7 +70,7 @@ BlockIO InterpreterDeleteQuery::execute() return database->tryEnqueueReplicatedDDL(query_ptr, getContext()); } - auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock = table->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(); if (table->supportsDelete()) @@ -86,7 +92,7 @@ BlockIO InterpreterDeleteQuery::execute() } else if (table->supportsLightweightDelete()) { - if (!getContext()->getSettingsRef().enable_lightweight_delete) + if (!getContext()->getSettingsRef()[Setting::enable_lightweight_delete]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Lightweight delete mutate is disabled. " "Set `enable_lightweight_delete` setting to enable it"); @@ -122,7 +128,7 @@ BlockIO InterpreterDeleteQuery::execute() DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); auto context = Context::createCopy(getContext()); - context->setSetting("mutations_sync", Field(context->getSettingsRef().lightweight_deletes_sync)); + context->setSetting("mutations_sync", Field(context->getSettingsRef()[Setting::lightweight_deletes_sync])); InterpreterAlterQuery alter_interpreter(alter_ast, context); return alter_interpreter.execute(); } diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 39fc85a5e23..2b494c548ff 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -23,6 +23,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool describe_compact_output; + extern const SettingsBool describe_extend_object_types; + extern const SettingsBool describe_include_subcolumns; + extern const SettingsBool describe_include_virtual_columns; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool print_pretty_type_names; +} InterpreterDescribeQuery::InterpreterDescribeQuery(const ASTPtr & query_ptr_, ContextPtr context_) : WithContext(context_) @@ -94,9 +104,7 @@ BlockIO InterpreterDescribeQuery::execute() fillColumnsFromTable(table_expression); Block sample_block = getSampleBlock( - settings.describe_include_subcolumns, - settings.describe_include_virtual_columns, - settings.describe_compact_output); + settings[Setting::describe_include_subcolumns], settings[Setting::describe_include_virtual_columns], settings[Setting::describe_compact_output]); MutableColumns res_columns = sample_block.cloneEmptyColumns(); @@ -106,7 +114,7 @@ BlockIO InterpreterDescribeQuery::execute() for (const auto & column : virtual_columns) addColumn(column, true, res_columns); - if (settings.describe_include_subcolumns) + if (settings[Setting::describe_include_subcolumns]) { for (const auto & column : columns) addSubcolumns(column, false, res_columns); @@ -129,7 +137,7 @@ void InterpreterDescribeQuery::fillColumnsFromSubquery(const ASTTableExpression auto select_query = table_expression.subquery->children.at(0); auto current_context = getContext(); - if (settings.allow_experimental_analyzer) + if (settings[Setting::allow_experimental_analyzer]) { SelectQueryOptions select_query_options; sample_block = InterpreterSelectQueryAnalyzer(select_query, current_context, select_query_options).getSampleBlock(); @@ -152,7 +160,7 @@ void InterpreterDescribeQuery::fillColumnsFromTableFunction(const ASTTableExpres for (const auto & column : column_descriptions) columns.emplace_back(column); - if (settings.describe_include_virtual_columns) + if (settings[Setting::describe_include_virtual_columns]) { auto table = table_function_ptr->execute(table_expression.table_function, getContext(), table_function_ptr->getName()); if (table) @@ -172,14 +180,14 @@ void InterpreterDescribeQuery::fillColumnsFromTable(const ASTTableExpression & t auto table_id = getContext()->resolveStorageID(table_expression.database_and_table_name); getContext()->checkAccess(AccessType::SHOW_COLUMNS, table_id); auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout); + auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(); const auto & column_descriptions = metadata_snapshot->getColumns(); for (const auto & column : column_descriptions) columns.emplace_back(column); - if (settings.describe_include_virtual_columns) + if (settings[Setting::describe_include_virtual_columns]) { auto virtuals = table->getVirtualsPtr(); for (const auto & column : *virtuals) @@ -189,7 +197,7 @@ void InterpreterDescribeQuery::fillColumnsFromTable(const ASTTableExpression & t } } - if (settings.describe_extend_object_types) + if (settings[Setting::describe_extend_object_types]) storage_snapshot = table->getStorageSnapshot(metadata_snapshot, getContext()); } @@ -199,12 +207,12 @@ void InterpreterDescribeQuery::addColumn(const ColumnDescription & column, bool res_columns[i++]->insert(column.name); auto type = storage_snapshot ? storage_snapshot->getConcreteType(column.name) : column.type; - if (settings.print_pretty_type_names) + if (settings[Setting::print_pretty_type_names]) res_columns[i++]->insert(type->getPrettyName()); else res_columns[i++]->insert(type->getName()); - if (!settings.describe_compact_output) + if (!settings[Setting::describe_compact_output]) { if (column.default_desc.expression) { @@ -230,10 +238,10 @@ void InterpreterDescribeQuery::addColumn(const ColumnDescription & column, bool res_columns[i++]->insertDefault(); } - if (settings.describe_include_subcolumns) + if (settings[Setting::describe_include_subcolumns]) res_columns[i++]->insertDefault(); - if (settings.describe_include_virtual_columns) + if (settings[Setting::describe_include_virtual_columns]) res_columns[i++]->insert(is_virtual); } @@ -246,12 +254,12 @@ void InterpreterDescribeQuery::addSubcolumns(const ColumnDescription & column, b size_t i = 0; res_columns[i++]->insert(Nested::concatenateName(column.name, name)); - if (settings.print_pretty_type_names) + if (settings[Setting::print_pretty_type_names]) res_columns[i++]->insert(data.type->getPrettyName()); else res_columns[i++]->insert(data.type->getName()); - if (!settings.describe_compact_output) + if (!settings[Setting::describe_compact_output]) { /// It's not trivial to calculate default expression for subcolumn. /// So, leave it empty. @@ -272,7 +280,7 @@ void InterpreterDescribeQuery::addSubcolumns(const ColumnDescription & column, b res_columns[i++]->insert(1U); - if (settings.describe_include_virtual_columns) + if (settings[Setting::describe_include_virtual_columns]) res_columns[i++]->insert(is_virtual); }, ISerialization::SubstreamData(type->getDefaultSerialization()).withType(type)); diff --git a/src/Interpreters/InterpreterDropIndexQuery.cpp b/src/Interpreters/InterpreterDropIndexQuery.cpp index 8777532e4d0..b843ca7acf8 100644 --- a/src/Interpreters/InterpreterDropIndexQuery.cpp +++ b/src/Interpreters/InterpreterDropIndexQuery.cpp @@ -12,6 +12,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -61,7 +65,7 @@ BlockIO InterpreterDropIndexQuery::execute() alter_commands.emplace_back(std::move(command)); - auto alter_lock = table->lockForAlter(current_context->getSettingsRef().lock_acquire_timeout); + auto alter_lock = table->lockForAlter(current_context->getSettingsRef()[Setting::lock_acquire_timeout]); StorageInMemoryMetadata metadata = table->getInMemoryMetadata(); alter_commands.validate(table, current_context); alter_commands.prepare(metadata); diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 5161fd19d87..1f4aa9fa1b7 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -30,6 +30,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool check_referential_table_dependencies; + extern const SettingsBool check_table_dependencies; + extern const SettingsBool database_atomic_wait_for_drop_and_detach_synchronously; + extern const SettingsFloat ignore_drop_queries_probability; + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -80,7 +88,7 @@ BlockIO InterpreterDropQuery::executeSingleDropQuery(const ASTPtr & drop_query_p return executeDDLQueryOnCluster(current_query_ptr, getContext(), params); } - if (getContext()->getSettingsRef().database_atomic_wait_for_drop_and_detach_synchronously) + if (getContext()->getSettingsRef()[Setting::database_atomic_wait_for_drop_and_detach_synchronously]) drop.sync = true; if (drop.table) @@ -165,7 +173,8 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS "Table {} is not a Dictionary", table_id.getNameForLogs()); - if (settings.ignore_drop_queries_probability != 0 && ast_drop_query.kind == ASTDropQuery::Kind::Drop && std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= settings.ignore_drop_queries_probability) + if (settings[Setting::ignore_drop_queries_probability] != 0 && ast_drop_query.kind == ASTDropQuery::Kind::Drop + && std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= settings[Setting::ignore_drop_queries_probability]) { ast_drop_query.sync = false; if (table->storesDataOnDisk()) @@ -239,13 +248,13 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS TableExclusiveLockHolder table_lock; if (database->getUUID() == UUIDHelpers::Nil) - table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); if (query.permanently) { /// Server may fail to restart of DETACH PERMANENTLY if table has dependent ones - bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; - bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; + bool check_ref_deps = getContext()->getSettingsRef()[Setting::check_referential_table_dependencies]; + bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef()[Setting::check_table_dependencies]; DatabaseCatalog::instance().removeDependencies(table_id, check_ref_deps, check_loading_deps, is_drop_or_detach_database); /// Drop table from memory, don't touch data, metadata file renamed and will be skipped during server restart database->detachTablePermanently(context_, table_id.table_name); @@ -271,7 +280,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS /// We don't need any lock for ReplicatedMergeTree and for simple MergeTree /// For the rest of tables types exclusive lock is needed if (!std::dynamic_pointer_cast(table)) - table_excl_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + table_excl_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Drop table data, don't touch metadata @@ -291,15 +300,15 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS table->checkTableCanBeDropped(context_); /// Check dependencies before shutting table down - bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; - bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; + bool check_ref_deps = getContext()->getSettingsRef()[Setting::check_referential_table_dependencies]; + bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef()[Setting::check_table_dependencies]; DatabaseCatalog::instance().checkTableCanBeRemovedOrRenamed(table_id, check_ref_deps, check_loading_deps, is_drop_or_detach_database); table->flushAndShutdown(true); TableExclusiveLockHolder table_lock; if (database->getUUID() == UUIDHelpers::Nil) - table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); DatabaseCatalog::instance().removeDependencies(table_id, check_ref_deps, check_loading_deps, is_drop_or_detach_database); database->dropTable(context_, table_id.table_name, query.sync); @@ -331,7 +340,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, if (kind == ASTDropQuery::Kind::Truncate) { auto table_lock - = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); /// Drop table data, don't touch metadata auto metadata_snapshot = table->getInMemoryMetadataPtr(); table->truncate(current_query_ptr, metadata_snapshot, getContext(), table_lock); diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index c820f999e0c..fdd138117d8 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -36,6 +36,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_statistics_optimize; +} namespace ErrorCodes { @@ -70,7 +75,7 @@ namespace static void visit(ASTSelectQuery & select, ASTPtr & node, Data & data) { /// we need to read statistic when `allow_statistics_optimize` is enabled. - bool only_analyze = !data.getContext()->getSettingsRef().allow_statistics_optimize; + bool only_analyze = !data.getContext()->getSettingsRef()[Setting::allow_statistics_optimize]; InterpreterSelectQuery interpreter( node, data.getContext(), SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze(only_analyze).modify()); @@ -394,7 +399,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() } case ASTExplainQuery::QueryTree: { - if (!getContext()->getSettingsRef().allow_experimental_analyzer) + if (!getContext()->getSettingsRef()[Setting::allow_experimental_analyzer]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "EXPLAIN QUERY TREE is only supported with a new analyzer. Set allow_experimental_analyzer = 1."); @@ -453,7 +458,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() ContextPtr context; - if (getContext()->getSettingsRef().allow_experimental_analyzer) + if (getContext()->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), options); context = interpreter.getContext(); @@ -499,7 +504,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() QueryPlan plan; ContextPtr context; - if (getContext()->getSettingsRef().allow_experimental_analyzer) + if (getContext()->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), options); context = interpreter.getContext(); @@ -558,7 +563,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl() QueryPlan plan; ContextPtr context = getContext(); - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), SelectQueryOptions()); context = interpreter.getContext(); diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp index 12b3b510098..cfc95124895 100644 --- a/src/Interpreters/InterpreterFactory.cpp +++ b/src/Interpreters/InterpreterFactory.cpp @@ -75,6 +75,12 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool insert_allow_materialized_columns; +} + namespace ErrorCodes { extern const int UNKNOWN_TYPE_OF_QUERY; @@ -118,7 +124,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte if (query->as()) { - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) interpreter_name = "InterpreterSelectQueryAnalyzer"; /// This is internal part of ASTSelectWithUnionQuery. /// Even if there is SELECT without union, it is represented by ASTSelectWithUnionQuery with single ASTSelectQuery as a child. @@ -129,7 +135,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte { ProfileEvents::increment(ProfileEvents::SelectQuery); - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) interpreter_name = "InterpreterSelectQueryAnalyzer"; else interpreter_name = "InterpreterSelectWithUnionQuery"; @@ -141,7 +147,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte else if (query->as()) { ProfileEvents::increment(ProfileEvents::InsertQuery); - bool allow_materialized = static_cast(context->getSettingsRef().insert_allow_materialized_columns); + bool allow_materialized = static_cast(context->getSettingsRef()[Setting::insert_allow_materialized_columns]); arguments.allow_materialized = allow_materialized; interpreter_name = "InterpreterInsertQuery"; } diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index c97593a1781..d7f9c338ab1 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -54,6 +54,24 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool distributed_foreground_insert; + extern const SettingsBool insert_null_as_default; + extern const SettingsBool optimize_trivial_insert_select; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 max_insert_threads; + extern const SettingsUInt64 min_insert_block_size_rows; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 preferred_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsString insert_deduplication_token; + extern const SettingsBool parallel_view_processing; + extern const SettingsBool use_concurrency_control; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 parallel_distributed_insert_select; +} namespace ErrorCodes { @@ -95,7 +113,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) Block header_block; auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1); - if (current_context->getSettingsRef().allow_experimental_analyzer) + if (current_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter_select(query.select, current_context, select_query_options); header_block = interpreter_select.getSampleBlock(); @@ -353,7 +371,7 @@ bool InterpreterInsertQuery::shouldAddSquashingFroStorage(const StoragePtr & tab /// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side. /// Client-side bufferization might cause excessive timeouts (especially in case of big blocks). - return !(settings.distributed_foreground_insert && table->isRemote()) && !async_insert && !no_squash; + return !(settings[Setting::distributed_foreground_insert] && table->isRemote()) && !async_insert && !no_squash; } Chain InterpreterInsertQuery::buildPreSinkChain( @@ -368,7 +386,7 @@ Chain InterpreterInsertQuery::buildPreSinkChain( if (query_ptr) query = query_ptr->as(); - bool null_as_default = query && query->select && context_ptr->getSettingsRef().insert_null_as_default; + bool null_as_default = query && query->select && context_ptr->getSettingsRef()[Setting::insert_null_as_default]; /// We create a pipeline of several streams, into which we will write data. Chain out; @@ -442,7 +460,7 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & bool is_trivial_insert_select = false; - if (settings.optimize_trivial_insert_select) + if (settings[Setting::optimize_trivial_insert_select]) { const auto & select_query = query.select->as(); const auto & selects = select_query.list_of_selects->children; @@ -468,14 +486,14 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & Settings new_settings = select_context->getSettingsCopy(); - new_settings.max_threads = std::max(1, settings.max_insert_threads); + new_settings[Setting::max_threads] = std::max(1, settings[Setting::max_insert_threads]); if (table->prefersLargeBlocks()) { - if (settings.min_insert_block_size_rows) - new_settings.max_block_size = settings.min_insert_block_size_rows; - if (settings.min_insert_block_size_bytes) - new_settings.preferred_block_size_bytes = settings.min_insert_block_size_bytes; + if (settings[Setting::min_insert_block_size_rows]) + new_settings[Setting::max_block_size] = settings[Setting::min_insert_block_size_rows]; + if (settings[Setting::min_insert_block_size_bytes]) + new_settings[Setting::preferred_block_size_bytes] = settings[Setting::min_insert_block_size_bytes]; } auto context_for_trivial_select = Context::createCopy(context); @@ -490,7 +508,7 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & { auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1); - if (settings.allow_experimental_analyzer) + if (settings[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, select_context, select_query_options); pipeline = interpreter_select_analyzer.buildQueryPipeline(); @@ -505,7 +523,7 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & pipeline.dropTotalsAndExtremes(); /// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values. - if (getContext()->getSettingsRef().insert_null_as_default) + if (getContext()->getSettingsRef()[Setting::insert_null_as_default]) { const auto & input_columns = pipeline.getHeader().getColumnsWithTypeAndName(); const auto & query_columns = query_sample_block.getColumnsWithTypeAndName(); @@ -568,13 +586,14 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & if (shouldAddSquashingFroStorage(table)) { - pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr - { - return std::make_shared( - in_header, - table->prefersLargeBlocks() ? settings.min_insert_block_size_rows : settings.max_block_size, - table->prefersLargeBlocks() ? settings.min_insert_block_size_bytes : 0ULL); - }); + pipeline.addSimpleTransform( + [&](const Block & in_header) -> ProcessorPtr + { + return std::make_shared( + in_header, + table->prefersLargeBlocks() ? settings[Setting::min_insert_block_size_rows] : settings[Setting::max_block_size], + table->prefersLargeBlocks() ? settings[Setting::min_insert_block_size_bytes] : 0ULL); + }); } pipeline.addSimpleTransform([&](const Block &in_header) -> ProcessorPtr @@ -582,11 +601,11 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & return std::make_shared(in_header); }); - if (!settings.insert_deduplication_token.value.empty()) + if (!settings[Setting::insert_deduplication_token].value.empty()) { pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr { - return std::make_shared(settings.insert_deduplication_token.value, in_header); + return std::make_shared(settings[Setting::insert_deduplication_token].value, in_header); }); pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr @@ -603,14 +622,14 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & /// * If the table supports parallel inserts, use max_insert_threads for writing to IStorage. /// Otherwise ResizeProcessor them down to 1 stream. - size_t presink_streams_size = std::max(settings.max_insert_threads, pipeline.getNumStreams()); - if (settings.max_insert_threads.changed) - presink_streams_size = std::max(1, settings.max_insert_threads); + size_t presink_streams_size = std::max(settings[Setting::max_insert_threads], pipeline.getNumStreams()); + if (settings[Setting::max_insert_threads].changed) + presink_streams_size = std::max(1, settings[Setting::max_insert_threads]); - size_t sink_streams_size = table->supportsParallelInsert() ? std::max(1, settings.max_insert_threads) : 1; + size_t sink_streams_size = table->supportsParallelInsert() ? std::max(1, settings[Setting::max_insert_threads]) : 1; size_t views_involved = table->isView() || !DatabaseCatalog::instance().getDependentViews(table->getStorageID()).empty(); - if (!settings.parallel_view_processing && views_involved) + if (!settings[Setting::parallel_view_processing] && views_involved) { sink_streams_size = 1; } @@ -623,13 +642,14 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & if (shouldAddSquashingFroStorage(table)) { - pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr - { - return std::make_shared( - in_header, - table->prefersLargeBlocks() ? settings.min_insert_block_size_rows : settings.max_block_size, - table->prefersLargeBlocks() ? settings.min_insert_block_size_bytes : 0ULL); - }); + pipeline.addSimpleTransform( + [&](const Block & in_header) -> ProcessorPtr + { + return std::make_shared( + in_header, + table->prefersLargeBlocks() ? settings[Setting::min_insert_block_size_rows] : settings[Setting::max_block_size], + table->prefersLargeBlocks() ? settings[Setting::min_insert_block_size_bytes] : 0ULL); + }); } for (auto & chain : presink_chains) @@ -642,7 +662,7 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & pipeline.addResources(chain.detachResources()); pipeline.addChains(std::move(sink_chains)); - if (!settings.parallel_view_processing && views_involved) + if (!settings[Setting::parallel_view_processing] && views_involved) { /// Don't use more threads for INSERT than for SELECT to reduce memory consumption. if (pipeline.getNumThreads() > num_select_threads) @@ -676,10 +696,11 @@ QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query chain.appendChain(std::move(sink_chains.front())); } - if (!settings.insert_deduplication_token.value.empty()) + if (!settings[Setting::insert_deduplication_token].value.empty()) { chain.addSource(std::make_shared(chain.getInputHeader())); - chain.addSource(std::make_shared(settings.insert_deduplication_token.value, chain.getInputHeader())); + chain.addSource(std::make_shared( + settings[Setting::insert_deduplication_token].value, chain.getInputHeader())); } chain.addSource(std::make_shared(chain.getInputHeader())); @@ -689,16 +710,16 @@ QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query bool table_prefers_large_blocks = table->prefersLargeBlocks(); auto squashing = std::make_shared( - chain.getInputHeader(), - table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size, - table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL); + chain.getInputHeader(), + table_prefers_large_blocks ? settings[Setting::min_insert_block_size_rows] : settings[Setting::max_block_size], + table_prefers_large_blocks ? settings[Setting::min_insert_block_size_bytes] : 0ULL); chain.addSource(std::move(squashing)); auto balancing = std::make_shared( - chain.getInputHeader(), - table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size, - table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL); + chain.getInputHeader(), + table_prefers_large_blocks ? settings[Setting::min_insert_block_size_rows] : settings[Setting::max_block_size], + table_prefers_large_blocks ? settings[Setting::min_insert_block_size_bytes] : 0ULL); chain.addSource(std::move(balancing)); } @@ -711,8 +732,8 @@ QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query QueryPipeline pipeline = QueryPipeline(std::move(chain)); - pipeline.setNumThreads(std::min(pipeline.getNumThreads(), settings.max_threads)); - pipeline.setConcurrencyControl(settings.use_concurrency_control); + pipeline.setNumThreads(std::min(pipeline.getNumThreads(), settings[Setting::max_threads])); + pipeline.setConcurrencyControl(settings[Setting::use_concurrency_control]); if (query.hasInlinedData() && !async_insert) { @@ -744,7 +765,7 @@ BlockIO InterpreterInsertQuery::execute() if (query.partition_by && !table->supportsPartitionBy()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PARTITION BY clause is not supported by storage"); - auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout); + auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(); auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized); @@ -765,7 +786,7 @@ BlockIO InterpreterInsertQuery::execute() if (query.select) { - if (settings.parallel_distributed_insert_select) + if (settings[Setting::parallel_distributed_insert_select]) { auto distributed = table->distributedWrite(query, getContext()); if (distributed) diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 2c579f3b468..b390b29c03c 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -32,6 +32,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -282,8 +287,12 @@ BlockIO InterpreterKillQueryQuery::execute() const auto alter_command = command_col.getDataAt(i).toString(); const auto with_round_bracket = alter_command.front() == '('; ParserAlterCommand parser{with_round_bracket}; - auto command_ast - = parseQuery(parser, alter_command, 0, getContext()->getSettingsRef().max_parser_depth, getContext()->getSettingsRef().max_parser_backtracks); + auto command_ast = parseQuery( + parser, + alter_command, + 0, + getContext()->getSettingsRef()[Setting::max_parser_depth], + getContext()->getSettingsRef()[Setting::max_parser_backtracks]); required_access_rights = InterpreterAlterQuery::getRequiredAccessForCommand( command_ast->as(), table_id.database_name, table_id.table_name); if (!access->isGranted(required_access_rights)) diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index c6e52590f89..ffa52b7ecf8 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool check_table_dependencies; + extern const SettingsBool check_referential_table_dependencies; +} namespace ErrorCodes { @@ -142,8 +147,8 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c else { DatabaseCatalog::instance().checkTableCanBeRenamedWithNoCyclicDependencies(from_table_id, to_table_id); - bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; - bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; + bool check_ref_deps = getContext()->getSettingsRef()[Setting::check_referential_table_dependencies]; + bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef()[Setting::check_table_dependencies]; std::tie(from_ref_dependencies, from_loading_dependencies) = database_catalog.removeDependencies(from_table_id, check_ref_deps, check_loading_deps); } diff --git a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp index b0da4caa3ac..ca23ff01036 100644 --- a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp +++ b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp @@ -22,6 +22,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsOverflowMode distinct_overflow_mode; + extern const SettingsUInt64 max_rows_in_distinct; + extern const SettingsUInt64 max_bytes_in_distinct; + extern const SettingsMaxThreads max_threads; + extern const SettingsBool optimize_distinct_in_order; +} namespace ErrorCodes { @@ -78,9 +86,8 @@ InterpreterSelectIntersectExceptQuery::InterpreterSelectIntersectExceptQuery( /// AST must have been changed by the visitor. if (final_operator == Operator::UNKNOWN || num_children != 2) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "SelectIntersectExceptyQuery has not been normalized (number of children: {})", - num_children); + throw Exception( + ErrorCodes::LOGICAL_ERROR, "SelectIntersectExceptQuery has not been normalized (number of children: {})", num_children); nested_interpreters.resize(num_children); @@ -143,8 +150,7 @@ void InterpreterSelectIntersectExceptQuery::buildQueryPlan(QueryPlan & query_pla } const Settings & settings = context->getSettingsRef(); - auto max_threads = settings.max_threads; - auto step = std::make_unique(std::move(data_streams), final_operator, max_threads); + auto step = std::make_unique(std::move(data_streams), final_operator, settings[Setting::max_threads]); query_plan.unitePlans(std::move(step), std::move(plans)); const auto & query = query_ptr->as(); @@ -152,7 +158,7 @@ void InterpreterSelectIntersectExceptQuery::buildQueryPlan(QueryPlan & query_pla || query.final_operator == ASTSelectIntersectExceptQuery::Operator::EXCEPT_DISTINCT) { /// Add distinct transform - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + SizeLimits limits(settings[Setting::max_rows_in_distinct], settings[Setting::max_bytes_in_distinct], settings[Setting::distinct_overflow_mode]); auto distinct_step = std::make_unique( query_plan.getCurrentDataStream(), @@ -160,7 +166,7 @@ void InterpreterSelectIntersectExceptQuery::buildQueryPlan(QueryPlan & query_pla 0, result_header.getNames(), false, - settings.optimize_distinct_in_order); + settings[Setting::optimize_distinct_in_order]); query_plan.addStep(std::move(distinct_step)); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index b7359b85079..259fddc6713 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -108,6 +108,86 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsMap additional_table_filters; + extern const SettingsUInt64 aggregation_in_order_max_block_bytes; + extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsBool allow_experimental_query_deduplication; + extern const SettingsBool async_socket_for_remote; + extern const SettingsBool collect_hash_table_stats_during_aggregation; + extern const SettingsBool compile_aggregate_expressions; + extern const SettingsBool compile_sort_description; + extern const SettingsBool count_distinct_optimization; + extern const SettingsUInt64 cross_to_inner_join_rewrite; + extern const SettingsOverflowMode distinct_overflow_mode; + extern const SettingsBool distributed_aggregation_memory_efficient; + extern const SettingsBool empty_result_for_aggregation_by_constant_keys_on_empty_set; + extern const SettingsBool empty_result_for_aggregation_by_empty_set; + extern const SettingsBool enable_global_with_statement; + extern const SettingsBool enable_memory_bound_merging_of_aggregation_results; + extern const SettingsBool enable_software_prefetch_in_aggregation; + extern const SettingsBool exact_rows_before_limit; + extern const SettingsBool enable_unaligned_array_join; + extern const SettingsBool extremes; + extern const SettingsBool final; + extern const SettingsBool force_aggregation_in_order; + extern const SettingsOverflowModeGroupBy group_by_overflow_mode; + extern const SettingsUInt64 group_by_two_level_threshold; + extern const SettingsUInt64 group_by_two_level_threshold_bytes; + extern const SettingsBool group_by_use_nulls; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_analyze_depth; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_bytes_before_external_group_by; + extern const SettingsUInt64 max_bytes_in_distinct; + extern const SettingsUInt64 max_columns_to_read; + extern const SettingsUInt64 max_distributed_connections; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsUInt64 max_result_bytes; + extern const SettingsUInt64 max_result_rows; + extern const SettingsUInt64 max_rows_in_distinct; + extern const SettingsUInt64 max_rows_in_set_to_optimize_join; + extern const SettingsUInt64 max_rows_to_group_by; + extern const SettingsUInt64 max_rows_to_read; + extern const SettingsUInt64 max_size_to_preallocate_for_aggregation; + extern const SettingsFloat max_streams_to_max_threads_ratio; + extern const SettingsUInt64 max_subquery_depth; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_count_to_compile_aggregate_expression; + extern const SettingsUInt64 min_count_to_compile_sort_description; + extern const SettingsUInt64 min_free_disk_space_for_temporary_data; + extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; + extern const SettingsBool multiple_joins_try_to_keep_original_names; + extern const SettingsBool optimize_aggregation_in_order; + extern const SettingsBool optimize_distinct_in_order; + extern const SettingsBool optimize_group_by_constant_keys; + extern const SettingsBool optimize_move_to_prewhere; + extern const SettingsBool optimize_move_to_prewhere_if_final; + extern const SettingsBool optimize_sorting_by_input_stream_properties; + extern const SettingsBool optimize_uniq_to_count; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsString parallel_replicas_custom_key; + extern const SettingsParallelReplicasCustomKeyFilterType parallel_replicas_custom_key_filter_type; + extern const SettingsUInt64 parallel_replicas_custom_key_range_lower; + extern const SettingsUInt64 parallel_replicas_custom_key_range_upper; + extern const SettingsBool parallel_replicas_for_non_replicated_merge_tree; + extern const SettingsUInt64 parallel_replicas_min_number_of_rows_per_replica; + extern const SettingsUInt64 parallel_replica_offset; + extern const SettingsBool query_plan_enable_optimizations; + extern const SettingsBool query_plan_enable_multithreading_after_window_functions; + extern const SettingsBool query_plan_optimize_prewhere; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; + extern const SettingsFloat totals_auto_threshold; + extern const SettingsTotalsMode totals_mode; + extern const SettingsBool use_concurrency_control; + extern const SettingsBool use_with_fill_by_sorting_prefix; +} + static UInt64 getLimitUIntValue(const ASTPtr & node, const ContextPtr & context, const std::string & expr); @@ -162,7 +242,12 @@ FilterDAGInfoPtr generateFilterActions( { ParserExpression expr_parser; /// We should add back quotes around column name as it can contain dots. - expr_list->children.push_back(parseQuery(expr_parser, backQuoteIfNeed(column_str), 0, context->getSettingsRef().max_parser_depth, context->getSettingsRef().max_parser_backtracks)); + expr_list->children.push_back(parseQuery( + expr_parser, + backQuoteIfNeed(column_str), + 0, + context->getSettingsRef()[Setting::max_parser_depth], + context->getSettingsRef()[Setting::max_parser_backtracks])); } select_ast->setExpression(ASTSelectQuery::Expression::TABLES, std::make_shared()); @@ -251,10 +336,10 @@ ContextPtr getSubqueryContext(const ContextPtr & context) { auto subquery_context = Context::createCopy(context); Settings subquery_settings = context->getSettingsCopy(); - subquery_settings.max_result_rows = 0; - subquery_settings.max_result_bytes = 0; + subquery_settings[Setting::max_result_rows] = 0; + subquery_settings[Setting::max_result_bytes] = 0; /// The calculation of extremes does not make sense and is not necessary (if you do it, then the extremes of the subquery can be taken for whole query). - subquery_settings.extremes = false; + subquery_settings[Setting::extremes] = false; subquery_context->setSettings(subquery_settings); return subquery_context; } @@ -269,11 +354,11 @@ void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, cons QueryAliasesNoSubqueriesVisitor(aliases).visit(select.select()); CrossToInnerJoinVisitor::Data cross_to_inner{tables, aliases, database}; - cross_to_inner.cross_to_inner_join_rewrite = static_cast(std::min(settings.cross_to_inner_join_rewrite, 2)); + cross_to_inner.cross_to_inner_join_rewrite = static_cast(std::min(settings[Setting::cross_to_inner_join_rewrite], 2)); CrossToInnerJoinVisitor(cross_to_inner).visit(query); JoinToSubqueryTransformVisitor::Data join_to_subs_data{tables, aliases}; - join_to_subs_data.try_to_keep_original_names = settings.multiple_joins_try_to_keep_original_names; + join_to_subs_data.try_to_keep_original_names = settings[Setting::multiple_joins_try_to_keep_original_names]; JoinToSubqueryTransformVisitor(join_to_subs_data).visit(query); } @@ -328,8 +413,13 @@ ASTPtr parseAdditionalFilterConditionForTable( ParserExpression parser; const auto & settings = context.getSettingsRef(); return parseQuery( - parser, filter.data(), filter.data() + filter.size(), - "additional filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parser, + filter.data(), + filter.data() + filter.size(), + "additional filter", + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); } } @@ -418,9 +508,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( initSettings(); const Settings & settings = context->getSettingsRef(); - if (settings.max_subquery_depth && options.subquery_depth > settings.max_subquery_depth) - throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, "Too deep subqueries. Maximum: {}", - settings.max_subquery_depth.toString()); + if (settings[Setting::max_subquery_depth] && options.subquery_depth > settings[Setting::max_subquery_depth]) + throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, "Too deep subqueries. Maximum: {}", settings[Setting::max_subquery_depth].toString()); bool has_input = input_pipe != std::nullopt; if (input_pipe) @@ -432,20 +521,20 @@ InterpreterSelectQuery::InterpreterSelectQuery( // Only propagate WITH elements to subqueries if we're not a subquery if (!options.is_subquery) { - if (context->getSettingsRef().enable_global_with_statement) + if (context->getSettingsRef()[Setting::enable_global_with_statement]) ApplyWithAliasVisitor::visit(query_ptr); ApplyWithSubqueryVisitor::visit(query_ptr); } query_info.query = query_ptr->clone(); - if (settings.count_distinct_optimization) + if (settings[Setting::count_distinct_optimization]) { RewriteCountDistinctFunctionMatcher::Data data_rewrite_countdistinct; RewriteCountDistinctFunctionVisitor(data_rewrite_countdistinct).visit(query_ptr); } - if (settings.optimize_uniq_to_count) + if (settings[Setting::optimize_uniq_to_count]) { RewriteUniqToCountMatcher::Data data_rewrite_uniq_count; RewriteUniqToCountVisitor(data_rewrite_uniq_count).visit(query_ptr); @@ -464,7 +553,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) { - table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + table_lock = storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); table_id = storage->getStorageID(); if (!metadata_snapshot) metadata_snapshot = storage->getInMemoryMetadataPtr(); @@ -478,7 +567,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (has_input || !joined_tables.resolveTables()) joined_tables.makeFakeTable(storage, metadata_snapshot, source_header); - if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (context->getCurrentTransaction() && context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) { if (storage) checkStorageSupportsTransactionsIfNeeded(storage, context, /* is_readonly_query */ true); @@ -494,7 +583,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( } /// Check support for JOIN for parallel replicas with custom key - if (joined_tables.tablesCount() > 1 && !settings.parallel_replicas_custom_key.value.empty()) + if (joined_tables.tablesCount() > 1 && !settings[Setting::parallel_replicas_custom_key].value.empty()) { LOG_DEBUG(log, "JOINs are not supported with parallel_replicas_custom_key. Query will be executed without using them."); context->setSetting("parallel_replicas_custom_key", String{""}); @@ -504,12 +593,12 @@ InterpreterSelectQuery::InterpreterSelectQuery( bool is_query_with_final = isQueryWithFinal(query_info); if (is_query_with_final && context->canUseTaskBasedParallelReplicas()) { - if (settings.allow_experimental_parallel_reading_from_replicas == 1) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] == 1) { LOG_DEBUG(log, "FINAL modifier is not supported with parallel replicas. Query will be executed without using them."); context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); } - else if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + else if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "FINAL modifier is not supported with parallel replicas"); } @@ -517,14 +606,15 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Check support for parallel replicas for non-replicated storage (plain MergeTree) bool is_plain_merge_tree = storage && storage->isMergeTree() && !storage->supportsReplication(); - if (is_plain_merge_tree && settings.allow_experimental_parallel_reading_from_replicas > 0 && !settings.parallel_replicas_for_non_replicated_merge_tree) + if (is_plain_merge_tree && settings[Setting::allow_experimental_parallel_reading_from_replicas] > 0 + && !settings[Setting::parallel_replicas_for_non_replicated_merge_tree]) { - if (settings.allow_experimental_parallel_reading_from_replicas == 1) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] == 1) { LOG_DEBUG(log, "To use parallel replicas with plain MergeTree tables please enable setting `parallel_replicas_for_non_replicated_merge_tree`. For now query will be executed without using them."); context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); } - else if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + else if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "To use parallel replicas with plain MergeTree tables please enable setting `parallel_replicas_for_non_replicated_merge_tree`"); } @@ -570,7 +660,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( joined_tables.rewriteDistributedInAndJoins(query_ptr); - max_streams = settings.max_threads; + max_streams = settings[Setting::max_threads]; ASTSelectQuery & query = getSelectQuery(); std::shared_ptr table_join = joined_tables.makeTableJoin(query); @@ -581,30 +671,30 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) view = dynamic_cast(storage.get()); - if (!settings.additional_table_filters.value.empty() && storage && !joined_tables.tablesWithColumns().empty()) + if (!settings[Setting::additional_table_filters].value.empty() && storage && !joined_tables.tablesWithColumns().empty()) query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( - settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); + settings[Setting::additional_table_filters], joined_tables.tablesWithColumns().front().table, *context); ASTPtr parallel_replicas_custom_filter_ast = nullptr; if (storage && context->canUseParallelReplicasCustomKey() && !joined_tables.tablesWithColumns().empty()) { - if (settings.parallel_replicas_count > 1) + if (settings[Setting::parallel_replicas_count] > 1) { - if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *context)) + if (auto custom_key_ast = parseCustomKeyForTable(settings[Setting::parallel_replicas_custom_key], *context)) { - LOG_TRACE(log, "Processing query on a replica using custom_key '{}'", settings.parallel_replicas_custom_key.value); + LOG_TRACE(log, "Processing query on a replica using custom_key '{}'", settings[Setting::parallel_replicas_custom_key].value); parallel_replicas_custom_filter_ast = getCustomKeyFilterForParallelReplica( - settings.parallel_replicas_count, - settings.parallel_replica_offset, + settings[Setting::parallel_replicas_count], + settings[Setting::parallel_replica_offset], std::move(custom_key_ast), - {settings.parallel_replicas_custom_key_filter_type, - settings.parallel_replicas_custom_key_range_lower, - settings.parallel_replicas_custom_key_range_upper}, + {settings[Setting::parallel_replicas_custom_key_filter_type], + settings[Setting::parallel_replicas_custom_key_range_lower], + settings[Setting::parallel_replicas_custom_key_range_upper]}, storage->getInMemoryMetadataPtr()->columns, context); } - else if (settings.parallel_replica_offset > 0) + else if (settings[Setting::parallel_replica_offset] > 0) { throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -624,7 +714,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( context->setSetting("prefer_localhost_replica", Field(0)); } else if ( - storage->isMergeTree() && (storage->supportsReplication() || settings.parallel_replicas_for_non_replicated_merge_tree) + storage->isMergeTree() && (storage->supportsReplication() || settings[Setting::parallel_replicas_for_non_replicated_merge_tree]) && context->getClientInfo().distributed_depth == 0 && context->canUseParallelReplicasCustomKeyForCluster(*context->getClusterForParallelReplicas())) { @@ -817,10 +907,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// This is a hack to make sure we reanalyze if GlobalSubqueriesVisitor changed allow_experimental_parallel_reading_from_replicas /// inside the query context (because it doesn't have write access to the main context) UInt64 parallel_replicas_before_analysis - = context->hasQueryContext() ? context->getQueryContext()->getSettingsRef().allow_experimental_parallel_reading_from_replicas : 0; + = context->hasQueryContext() ? context->getQueryContext()->getSettingsRef()[Setting::allow_experimental_parallel_reading_from_replicas] : 0; /// Conditionally support AST-based PREWHERE optimization. - analyze(shouldMoveToPrewhere() && (!settings.query_plan_optimize_prewhere || !settings.query_plan_enable_optimizations)); + analyze(shouldMoveToPrewhere() && (!settings[Setting::query_plan_optimize_prewhere] || !settings[Setting::query_plan_enable_optimizations])); bool need_analyze_again = false; @@ -829,7 +919,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (context->hasQueryContext()) { /// As this query can't be executed with parallel replicas, we must reanalyze it - if (context->getQueryContext()->getSettingsRef().allow_experimental_parallel_reading_from_replicas + if (context->getQueryContext()->getSettingsRef()[Setting::allow_experimental_parallel_reading_from_replicas] != parallel_replicas_before_analysis) { context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); @@ -841,7 +931,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// If it is too big, we will not analyze the query again not to have exponential blowup. std::atomic & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; ++current_query_analyze_count; - can_analyze_again = settings.max_analyze_depth == 0 || current_query_analyze_count < settings.max_analyze_depth; + can_analyze_again = settings[Setting::max_analyze_depth] == 0 || current_query_analyze_count < settings[Setting::max_analyze_depth]; } if (can_analyze_again && (analysis_result.prewhere_constant_filter_description.always_false || @@ -931,7 +1021,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() } auto storage_merge_tree = std::dynamic_pointer_cast(storage); - if (!storage_merge_tree || settings.parallel_replicas_min_number_of_rows_per_replica == 0) + if (!storage_merge_tree || settings[Setting::parallel_replicas_min_number_of_rows_per_replica] == 0) return false; auto query_info_copy = query_info; @@ -941,8 +1031,8 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() /// * The max_rows_to_read setting /// * A LIMIT in a simple query (see maxBlockSizeByLimit()) UInt64 max_rows = maxBlockSizeByLimit(); - if (settings.max_rows_to_read) - max_rows = max_rows ? std::min(max_rows, settings.max_rows_to_read.value) : settings.max_rows_to_read; + if (settings[Setting::max_rows_to_read]) + max_rows = max_rows ? std::min(max_rows, settings[Setting::max_rows_to_read].value) : settings[Setting::max_rows_to_read]; query_info_copy.trivial_limit = max_rows; /// Apply filters to prewhere and add them to the query_info so we can filter out parts efficiently during row estimation @@ -975,7 +1065,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() query_info_copy.filter_actions_dag = std::make_shared(std::move(*filter_actions_dag)); UInt64 rows_to_read = storage_merge_tree->estimateNumberOfRowsToRead(context, storage_snapshot, query_info_copy); /// Note that we treat an estimation of 0 rows as a real estimation - size_t number_of_replicas_to_use = rows_to_read / settings.parallel_replicas_min_number_of_rows_per_replica; + size_t number_of_replicas_to_use = rows_to_read / settings[Setting::parallel_replicas_min_number_of_rows_per_replica]; LOG_TRACE(log, "Estimated {} rows to read. It is enough work for {} parallel replicas", rows_to_read, number_of_replicas_to_use); if (number_of_replicas_to_use <= 1) @@ -985,7 +1075,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() LOG_DEBUG(log, "Disabling parallel replicas because there aren't enough rows to read"); return true; } - else if (number_of_replicas_to_use < settings.max_parallel_replicas) + else if (number_of_replicas_to_use < settings[Setting::max_parallel_replicas]) { context->setSetting("max_parallel_replicas", number_of_replicas_to_use); LOG_DEBUG(log, "Reducing the number of replicas to use to {}", number_of_replicas_to_use); @@ -1126,7 +1216,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() if (analysis_result.use_grouping_set_key) res.insert({ nullptr, std::make_shared(), "__grouping_set" }); - if (context->getSettingsRef().group_by_use_nulls && analysis_result.use_grouping_set_key) + if (context->getSettingsRef()[Setting::group_by_use_nulls] && analysis_result.use_grouping_set_key) { for (const auto & key : query_analyzer->aggregationKeys()) res.insert({nullptr, makeNullableSafe(header.getByName(key.name).type), key.name}); @@ -1265,8 +1355,8 @@ SortDescription InterpreterSelectQuery::getSortDescription(const ASTSelectQuery order_descr.emplace_back(column_name, order_by_elem.direction, order_by_elem.nulls_direction, collator); } - order_descr.compile_sort_description = context_->getSettingsRef().compile_sort_description; - order_descr.min_count_to_compile_sort_description = context_->getSettingsRef().min_count_to_compile_sort_description; + order_descr.compile_sort_description = context_->getSettingsRef()[Setting::compile_sort_description]; + order_descr.min_count_to_compile_sort_description = context_->getSettingsRef()[Setting::min_count_to_compile_sort_description]; return order_descr; } @@ -1486,9 +1576,9 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( query_plan.getCurrentDataStream(), *expressions.array_join, - settings.enable_unaligned_array_join, - settings.max_block_size); + settings[Setting::enable_unaligned_array_join], + settings[Setting::max_block_size]); array_join_step->setStepDescription("ARRAY JOIN"); query_plan.addStep(std::move(array_join_step)); @@ -1698,10 +1788,8 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

isFilled()) { - QueryPlanStepPtr filled_join_step = std::make_unique( - query_plan.getCurrentDataStream(), - expressions.join, - settings.max_block_size); + QueryPlanStepPtr filled_join_step + = std::make_unique(query_plan.getCurrentDataStream(), expressions.join, settings[Setting::max_block_size]); filled_join_step->setStepDescription("JOIN"); query_plan.addStep(std::move(filled_join_step)); @@ -1725,17 +1813,23 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( plan.getCurrentDataStream(), std::move(order_descr), - 0 /* LIMIT */, sort_settings, - settings.optimize_sorting_by_input_stream_properties); + 0 /* LIMIT */, + sort_settings, + settings[Setting::optimize_sorting_by_input_stream_properties]); sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", join_pos)); plan.addStep(std::move(sorting_step)); }; auto crosswise_connection = CreateSetAndFilterOnTheFlyStep::createCrossConnection(); - auto add_create_set = [&settings, crosswise_connection](QueryPlan & plan, const Names & key_names, JoinTableSide join_pos) + auto add_create_set + = [&settings, crosswise_connection](QueryPlan & plan, const Names & key_names, JoinTableSide join_pos) { auto creating_set_step = std::make_unique( - plan.getCurrentDataStream(), key_names, settings.max_rows_in_set_to_optimize_join, crosswise_connection, join_pos); + plan.getCurrentDataStream(), + key_names, + settings[Setting::max_rows_in_set_to_optimize_join], + crosswise_connection, + join_pos); creating_set_step->setStepDescription(fmt::format("Create set and filter {} joined stream", join_pos)); auto * step_raw_ptr = creating_set_step.get(); @@ -1771,7 +1865,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

getCurrentDataStream().header, join_clause.key_names_right); - if (settings.max_rows_in_set_to_optimize_join > 0 && join_type_allows_filtering && has_non_const_keys) + if (settings[Setting::max_rows_in_set_to_optimize_join] > 0 && join_type_allows_filtering && has_non_const_keys) { auto * left_set = add_create_set(query_plan, join_clause.key_names_left, JoinTableSide::Left); auto * right_set = add_create_set(*joined_plan, join_clause.key_names_right, JoinTableSide::Right); @@ -1791,7 +1885,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

getCurrentDataStream(), expressions.join, - settings.max_block_size, + settings[Setting::max_block_size], max_streams, analysis_result.optimize_read_in_order); @@ -1965,7 +2059,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( @@ -2062,14 +2162,14 @@ static void executeMergeAggregatedImpl( grouping_sets_params, final, /// Grouping sets don't work with distributed_aggregation_memory_efficient enabled (#43989) - settings.distributed_aggregation_memory_efficient && is_remote_storage && !has_grouping_sets, - settings.max_threads, - settings.aggregation_memory_efficient_merge_threads, + settings[Setting::distributed_aggregation_memory_efficient] && is_remote_storage && !has_grouping_sets, + settings[Setting::max_threads], + settings[Setting::aggregation_memory_efficient_merge_threads], should_produce_results_in_order_of_bucket_number, - settings.max_block_size, - settings.aggregation_in_order_max_block_bytes, + settings[Setting::max_block_size], + settings[Setting::aggregation_in_order_max_block_bytes], std::move(group_by_sort_description), - settings.enable_memory_bound_merging_of_aggregation_results); + settings[Setting::enable_memory_bound_merging_of_aggregation_results]); query_plan.addStep(std::move(merging_aggregated)); } @@ -2125,7 +2225,7 @@ bool InterpreterSelectQuery::shouldMoveToPrewhere() const { const Settings & settings = context->getSettingsRef(); const ASTSelectQuery & query = query_ptr->as(); - return settings.optimize_move_to_prewhere && (!query.final() || settings.optimize_move_to_prewhere_if_final); + return settings[Setting::optimize_move_to_prewhere] && (!query.final() || settings[Setting::optimize_move_to_prewhere_if_final]); } /// Note that this is const and accepts the analysis ref to be able to use it to do analysis for parallel replicas @@ -2325,8 +2425,8 @@ std::optional InterpreterSelectQuery::getTrivialCount(UInt64 max_paralle bool optimize_trivial_count = syntax_analyzer_result->optimize_trivial_count && (max_parallel_replicas <= 1) - && !settings.allow_experimental_query_deduplication - && !settings.empty_result_for_aggregation_by_empty_set + && !settings[Setting::allow_experimental_query_deduplication] + && !settings[Setting::empty_result_for_aggregation_by_empty_set] && storage && storage->supportsTrivialCountOptimization(storage_snapshot, getContext()) && query_info.filter_asts.empty() @@ -2408,7 +2508,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc std::optional num_rows; /// Optimization for trivial query like SELECT count() FROM table. - if (processing_stage == QueryProcessingStage::FetchColumns && (num_rows = getTrivialCount(settings.max_parallel_replicas))) + if (processing_stage == QueryProcessingStage::FetchColumns && (num_rows = getTrivialCount(settings[Setting::max_parallel_replicas]))) { const auto & desc = query_analyzer->aggregates()[0]; const auto & func = desc.function; @@ -2446,15 +2546,15 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc /// Limitation on the number of columns to read. /// It's not applied in 'only_analyze' mode, because the query could be analyzed without removal of unnecessary columns. - if (!options.only_analyze && settings.max_columns_to_read && required_columns.size() > settings.max_columns_to_read) + if (!options.only_analyze && settings[Setting::max_columns_to_read] && required_columns.size() > settings[Setting::max_columns_to_read]) throw Exception( ErrorCodes::TOO_MANY_COLUMNS, "Limit for number of columns to read exceeded. Requested: {}, maximum: {}", required_columns.size(), - settings.max_columns_to_read); + settings[Setting::max_columns_to_read]); /// General limit for the number of threads. - size_t max_threads_execute_query = settings.max_threads; + size_t max_threads_execute_query = settings[Setting::max_threads]; /** * To simultaneously query more remote servers when async_socket_for_remote is off @@ -2469,13 +2569,13 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc * */ bool is_sync_remote = false; - if (storage && storage->isRemote() && !settings.async_socket_for_remote) + if (storage && storage->isRemote() && !settings[Setting::async_socket_for_remote]) { is_sync_remote = true; - max_threads_execute_query = max_streams = settings.max_distributed_connections; + max_threads_execute_query = max_streams = settings[Setting::max_distributed_connections]; } - UInt64 max_block_size = settings.max_block_size; + UInt64 max_block_size = settings[Setting::max_block_size]; auto local_limits = getStorageLimits(*context, options); if (UInt64 max_block_limited = maxBlockSizeByLimit()) @@ -2537,7 +2637,8 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc /// If necessary, we request more sources than the number of threads - to distribute the work evenly over the threads. if (max_streams > 1 && !is_sync_remote) { - if (auto streams_with_ratio = max_streams * settings.max_streams_to_max_threads_ratio; canConvertTo(streams_with_ratio)) + if (auto streams_with_ratio = max_streams * settings[Setting::max_streams_to_max_threads_ratio]; + canConvertTo(streams_with_ratio)) max_streams = static_cast(streams_with_ratio); else throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, @@ -2552,7 +2653,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc query_info.prewhere_info = prewhere_info; bool optimize_read_in_order = analysis_result.optimize_read_in_order; - bool optimize_aggregation_in_order = analysis_result.optimize_aggregation_in_order && !query_analyzer->useGroupingSetKey(); + bool optimize_aggregation_in_order = analysis_result.optimize_read_in_order && !query_analyzer->useGroupingSetKey(); /// Create optimizer with prepared actions. /// Maybe we will need to calc input_order_info later, e.g. while reading from StorageMerge. @@ -2612,7 +2713,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (!query_plan.getMaxThreads() || is_sync_remote) query_plan.setMaxThreads(max_threads_execute_query); - query_plan.setConcurrencyControl(settings.use_concurrency_control); + query_plan.setConcurrencyControl(settings[Setting::use_concurrency_control]); /// Aliases in table declaration. if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions) @@ -2649,38 +2750,42 @@ static Aggregator::Params getAggregatorParams( { const auto stats_collecting_params = StatsCollectingParams( calculateCacheKey(query_ptr), - settings.collect_hash_table_stats_during_aggregation, + settings[Setting::collect_hash_table_stats_during_aggregation], context.getServerSettings().max_entries_for_hash_table_stats, - settings.max_size_to_preallocate_for_aggregation); + settings[Setting::max_size_to_preallocate_for_aggregation]); return Aggregator::Params { keys, aggregates, overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, + settings[Setting::max_rows_to_group_by], + settings[Setting::group_by_overflow_mode], group_by_two_level_threshold, group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + settings[Setting::max_bytes_before_external_group_by], + settings[Setting::empty_result_for_aggregation_by_empty_set] + || (settings[Setting::empty_result_for_aggregation_by_constant_keys_on_empty_set] && keys.empty() && query_analyzer.hasConstAggregationKeys()), context.getTempDataOnDisk(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression, - settings.max_block_size, - settings.enable_software_prefetch_in_aggregation, + settings[Setting::max_threads], + settings[Setting::min_free_disk_space_for_temporary_data], + settings[Setting::compile_aggregate_expressions], + settings[Setting::min_count_to_compile_aggregate_expression], + settings[Setting::max_block_size], + settings[Setting::enable_software_prefetch_in_aggregation], /* only_merge */ false, - settings.optimize_group_by_constant_keys, - settings.min_hit_rate_to_use_consecutive_keys_optimization, - stats_collecting_params - }; + settings[Setting::optimize_group_by_constant_keys], + settings[Setting::min_hit_rate_to_use_consecutive_keys_optimization], + stats_collecting_params}; } -void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool overflow_row, bool final, InputOrderInfoPtr group_by_info) +void InterpreterSelectQuery::executeAggregation( + QueryPlan & query_plan, + const ActionsAndProjectInputsFlagPtr & expression, + bool overflow_row, + bool final, + InputOrderInfoPtr group_by_info) { executeExpression(query_plan, expression, "Before GROUP BY"); @@ -2696,15 +2801,15 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac aggregates, overflow_row, settings, - settings.group_by_two_level_threshold, - settings.group_by_two_level_threshold_bytes); + settings[Setting::group_by_two_level_threshold], + settings[Setting::group_by_two_level_threshold_bytes]); auto grouping_sets_params = getAggregatorGroupingSetsParams(query_analyzer->aggregationKeysList(), keys); SortDescription group_by_sort_description; SortDescription sort_description_for_merging; - if (group_by_info && settings.optimize_aggregation_in_order && !query_analyzer->useGroupingSetKey()) + if (group_by_info && settings[Setting::optimize_aggregation_in_order] && !query_analyzer->useGroupingSetKey()) { group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); sort_description_for_merging = group_by_info->sort_description_for_merging; @@ -2712,38 +2817,38 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac else group_by_info = nullptr; - if (!group_by_info && settings.force_aggregation_in_order) + if (!group_by_info && settings[Setting::force_aggregation_in_order]) { group_by_sort_description = getSortDescriptionFromGroupBy(getSelectQuery()); sort_description_for_merging = group_by_sort_description; } auto merge_threads = max_streams; - auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads - ? static_cast(settings.aggregation_memory_efficient_merge_threads) - : static_cast(settings.max_threads); + auto temporary_data_merge_threads = settings[Setting::aggregation_memory_efficient_merge_threads] + ? static_cast(settings[Setting::aggregation_memory_efficient_merge_threads]) + : static_cast(settings[Setting::max_threads]); bool storage_has_evenly_distributed_read = storage && storage->hasEvenlyDistributedRead(); const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState - && (settings.distributed_aggregation_memory_efficient || settings.enable_memory_bound_merging_of_aggregation_results); + && (settings[Setting::distributed_aggregation_memory_efficient] || settings[Setting::enable_memory_bound_merging_of_aggregation_results]); auto aggregating_step = std::make_unique( query_plan.getCurrentDataStream(), std::move(aggregator_params), std::move(grouping_sets_params), final, - settings.max_block_size, - settings.aggregation_in_order_max_block_bytes, + settings[Setting::max_block_size], + settings[Setting::aggregation_in_order_max_block_bytes], merge_threads, temporary_data_merge_threads, storage_has_evenly_distributed_read, - settings.group_by_use_nulls, + settings[Setting::group_by_use_nulls], std::move(sort_description_for_merging), std::move(group_by_sort_description), should_produce_results_in_order_of_bucket_number, - settings.enable_memory_bound_merging_of_aggregation_results, - !group_by_info && settings.force_aggregation_in_order); + settings[Setting::enable_memory_bound_merging_of_aggregation_results], + !group_by_info && settings[Setting::force_aggregation_in_order]); query_plan.addStep(std::move(aggregating_step)); } @@ -2756,7 +2861,7 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool = !query_analyzer->useGroupingSetKey() ? getSortDescriptionFromGroupBy(getSelectQuery()) : SortDescription{}; const bool should_produce_results_in_order_of_bucket_number = options.to_stage == QueryProcessingStage::WithMergeableState - && (settings.distributed_aggregation_memory_efficient || settings.enable_memory_bound_merging_of_aggregation_results); + && (settings[Setting::distributed_aggregation_memory_efficient] || settings[Setting::enable_memory_bound_merging_of_aggregation_results]); const bool parallel_replicas_from_merge_tree = storage->isMergeTree() && context->canUseParallelReplicasOnInitiator(); executeMergeAggregatedImpl( @@ -2789,7 +2894,12 @@ void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const Actions void InterpreterSelectQuery::executeTotalsAndHaving( - QueryPlan & query_plan, bool has_having, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter, bool overflow_row, bool final) + QueryPlan & query_plan, + bool has_having, + const ActionsAndProjectInputsFlagPtr & expression, + bool remove_filter, + bool overflow_row, + bool final) { std::optional dag; if (expression) @@ -2808,8 +2918,8 @@ void InterpreterSelectQuery::executeTotalsAndHaving( std::move(dag), has_having ? getSelectQuery().having()->getColumnName() : "", remove_filter, - settings.totals_mode, - settings.totals_auto_threshold, + settings[Setting::totals_mode], + settings[Setting::totals_auto_threshold], final); query_plan.addStep(std::move(totals_having_step)); @@ -2831,9 +2941,9 @@ void InterpreterSelectQuery::executeRollupOrCube(QueryPlan & query_plan, Modific QueryPlanStepPtr step; if (modificator == Modificator::ROLLUP) - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final, settings.group_by_use_nulls); + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final, settings[Setting::group_by_use_nulls]); else if (modificator == Modificator::CUBE) - step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final, settings.group_by_use_nulls); + step = std::make_unique(query_plan.getCurrentDataStream(), std::move(params), final, settings[Setting::group_by_use_nulls]); query_plan.addStep(std::move(step)); } @@ -2924,7 +3034,7 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) if (need_sort && i != 0) { need_sort = !sortIsPrefix(window, *windows_sorted[i - 1]) - || (settings.max_threads != 1 && window.partition_by.size() != windows_sorted[i - 1]->partition_by.size()); + || (settings[Setting::max_threads] != 1 && window.partition_by.size() != windows_sorted[i - 1]->partition_by.size()); } if (need_sort) { @@ -2936,14 +3046,15 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) window.partition_by, 0 /* LIMIT */, sort_settings, - settings.optimize_sorting_by_input_stream_properties); + settings[Setting::optimize_sorting_by_input_stream_properties]); sorting_step->setStepDescription("Sorting for window '" + window.window_name + "'"); query_plan.addStep(std::move(sorting_step)); } // Fan out streams only for the last window to preserve the ordering between windows, // and WindowTransform works on single stream anyway. - const bool streams_fan_out = settings.query_plan_enable_multithreading_after_window_functions && ((i + 1) == windows_sorted.size()); + const bool streams_fan_out + = settings[Setting::query_plan_enable_multithreading_after_window_functions] && ((i + 1) == windows_sorted.size()); auto window_step = std::make_unique(query_plan.getCurrentDataStream(), window, window.window_functions, streams_fan_out); window_step->setStepDescription("Window step for window '" + window.window_name + "'"); @@ -2961,7 +3072,7 @@ void InterpreterSelectQuery::executeOrderOptimized(QueryPlan & query_plan, Input query_plan.getCurrentDataStream(), input_sorting_info->sort_description_for_merging, output_order_descr, - settings.max_block_size, + settings[Setting::max_block_size], limit); query_plan.addStep(std::move(finish_sorting_step)); @@ -2995,7 +3106,7 @@ void InterpreterSelectQuery::executeOrder(QueryPlan & query_plan, InputOrderInfo output_order_descr, limit, sort_settings, - settings.optimize_sorting_by_input_stream_properties); + settings[Setting::optimize_sorting_by_input_stream_properties]); sorting_step->setStepDescription("Sorting for ORDER BY"); query_plan.addStep(std::move(sorting_step)); @@ -3007,8 +3118,8 @@ void InterpreterSelectQuery::executeMergeSorted(QueryPlan & query_plan, const st const auto & query = getSelectQuery(); SortDescription sort_description = getSortDescription(query, context); const UInt64 limit = getLimitForSorting(query, context); - const auto max_block_size = context->getSettingsRef().max_block_size; - const auto exact_rows_before_limit = context->getSettingsRef().exact_rows_before_limit; + const auto max_block_size = context->getSettingsRef()[Setting::max_block_size]; + const auto exact_rows_before_limit = context->getSettingsRef()[Setting::exact_rows_before_limit]; auto merging_sorted = std::make_unique( query_plan.getCurrentDataStream(), std::move(sort_description), max_block_size, limit, exact_rows_before_limit); @@ -3043,7 +3154,7 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before limit_for_distinct = limit_length + limit_offset; } - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + SizeLimits limits(settings[Setting::max_rows_in_distinct], settings[Setting::max_bytes_in_distinct], settings[Setting::distinct_overflow_mode]); auto distinct_step = std::make_unique( query_plan.getCurrentDataStream(), @@ -3051,7 +3162,7 @@ void InterpreterSelectQuery::executeDistinct(QueryPlan & query_plan, bool before limit_for_distinct, columns, pre_distinct, - settings.optimize_distinct_in_order); + settings[Setting::optimize_distinct_in_order]); if (pre_distinct) distinct_step->setStepDescription("Preliminary DISTINCT"); @@ -3081,7 +3192,8 @@ void InterpreterSelectQuery::executePreLimit(QueryPlan & query_plan, bool do_not const Settings & settings = context->getSettingsRef(); - auto limit = std::make_unique(query_plan.getCurrentDataStream(), limit_length, limit_offset, settings.exact_rows_before_limit); + auto limit + = std::make_unique(query_plan.getCurrentDataStream(), limit_length, limit_offset, settings[Setting::exact_rows_before_limit]); if (do_not_skip_offset) limit->setStepDescription("preliminary LIMIT (with OFFSET)"); else @@ -3134,7 +3246,7 @@ void InterpreterSelectQuery::executeWithFill(QueryPlan & query_plan) std::move(sort_description), std::move(fill_description), interpolate_descr, - settings.use_with_fill_by_sorting_prefix); + settings[Setting::use_with_fill_by_sorting_prefix]); query_plan.addStep(std::move(filling_step)); } } @@ -3156,7 +3268,7 @@ void InterpreterSelectQuery::executeLimit(QueryPlan & query_plan) * otherwise TOTALS is counted according to incomplete data. */ const Settings & settings = context->getSettingsRef(); - bool always_read_till_end = settings.exact_rows_before_limit; + bool always_read_till_end = settings[Setting::exact_rows_before_limit]; if (query.group_by_with_totals && !query.orderBy()) always_read_till_end = true; @@ -3205,7 +3317,7 @@ void InterpreterSelectQuery::executeOffset(QueryPlan & query_plan) void InterpreterSelectQuery::executeExtremes(QueryPlan & query_plan) { - if (!context->getSettingsRef().extremes) + if (!context->getSettingsRef()[Setting::extremes]) return; auto extremes_step = std::make_unique(query_plan.getCurrentDataStream()); @@ -3236,7 +3348,7 @@ void InterpreterSelectQuery::ignoreWithTotals() bool InterpreterSelectQuery::autoFinalOnQuery(ASTSelectQuery & query) { // query.tables() is required because not all queries have tables in it, it could be a function. - bool is_auto_final_setting_on = context->getSettingsRef().final; + bool is_auto_final_setting_on = context->getSettingsRef()[Setting::final]; bool is_final_supported = storage && storage->supportsFinal() && !storage->isRemote() && query.tables(); bool is_query_already_final = query.final(); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index e953a07b41d..2b00ab030d3 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -31,6 +31,17 @@ namespace DB { +namespace Setting +{ + extern const SettingsOverflowMode distinct_overflow_mode; + extern const SettingsBool exact_rows_before_limit; + extern const SettingsUInt64 limit; + extern const SettingsUInt64 max_bytes_in_distinct; + extern const SettingsUInt64 max_rows_in_distinct; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 offset; + extern const SettingsBool optimize_distinct_in_order; +} namespace ErrorCodes { @@ -52,7 +63,7 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( bool require_full_header = ast->hasNonDefaultUnionMode(); const Settings & settings = context->getSettingsRef(); - if (options.subquery_depth == 0 && (settings.limit > 0 || settings.offset > 0)) + if (options.subquery_depth == 0 && (settings[Setting::limit] > 0 || settings[Setting::offset] > 0)) settings_limit_offset_needed = true; size_t num_children = ast->list_of_selects->children.size(); @@ -110,13 +121,13 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (limit_offset_ast) { limit_offset = evaluateConstantExpressionAsLiteral(limit_offset_ast, context)->as().value.safeGet(); - UInt64 new_limit_offset = settings.offset + limit_offset; + UInt64 new_limit_offset = settings[Setting::offset] + limit_offset; ASTPtr new_limit_offset_ast = std::make_shared(new_limit_offset); select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, std::move(new_limit_offset_ast)); } - else if (settings.offset) + else if (settings[Setting::offset]) { - ASTPtr new_limit_offset_ast = std::make_shared(settings.offset.value); + ASTPtr new_limit_offset_ast = std::make_shared(settings[Setting::offset].value); select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, std::move(new_limit_offset_ast)); } @@ -126,18 +137,18 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( limit_length = evaluateConstantExpressionAsLiteral(limit_length_ast, context)->as().value.safeGet(); UInt64 new_limit_length = 0; - if (settings.offset == 0) - new_limit_length = std::min(limit_length, settings.limit.value); - else if (settings.offset < limit_length) - new_limit_length = settings.limit ? std::min(settings.limit.value, limit_length - settings.offset.value) - : (limit_length - settings.offset.value); + if (settings[Setting::offset] == 0) + new_limit_length = std::min(limit_length, settings[Setting::limit].value); + else if (settings[Setting::offset] < limit_length) + new_limit_length = settings[Setting::limit] ? std::min(settings[Setting::limit].value, limit_length - settings[Setting::offset].value) + : (limit_length - settings[Setting::offset].value); ASTPtr new_limit_length_ast = std::make_shared(new_limit_length); select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, std::move(new_limit_length_ast)); } - else if (settings.limit) + else if (settings[Setting::limit]) { - ASTPtr new_limit_length_ast = std::make_shared(settings.limit.value); + ASTPtr new_limit_length_ast = std::make_shared(settings[Setting::limit].value); select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, std::move(new_limit_length_ast)); } @@ -171,7 +182,7 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); /// Here we check that, in case if required_result_column_names were specified, /// nested interpreter returns exactly it. Except if query requires full header. - /// The code aboew is written in a way that for 0th query required_result_column_names_for_other_selects[0] + /// The code above is written in a way that for 0th query required_result_column_names_for_other_selects[0] /// is an empty list, and we should use required_result_column_names instead. const auto & current_required_result_column_names = (query_num == 0 && !require_full_header) ? required_result_column_names @@ -328,7 +339,7 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) data_streams[i] = plans[i]->getCurrentDataStream(); } - auto max_threads = settings.max_threads; + auto max_threads = settings[Setting::max_threads]; auto union_step = std::make_unique(std::move(data_streams), max_threads); query_plan.unitePlans(std::move(union_step), std::move(plans)); @@ -337,15 +348,10 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) if (query.union_mode == SelectUnionMode::UNION_DISTINCT) { /// Add distinct transform - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + SizeLimits limits(settings[Setting::max_rows_in_distinct], settings[Setting::max_bytes_in_distinct], settings[Setting::distinct_overflow_mode]); auto distinct_step = std::make_unique( - query_plan.getCurrentDataStream(), - limits, - 0, - result_header.getNames(), - false, - settings.optimize_distinct_in_order); + query_plan.getCurrentDataStream(), limits, 0, result_header.getNames(), false, settings[Setting::optimize_distinct_in_order]); query_plan.addStep(std::move(distinct_step)); } @@ -353,15 +359,16 @@ void InterpreterSelectWithUnionQuery::buildQueryPlan(QueryPlan & query_plan) if (settings_limit_offset_needed && !options.settings_limit_offset_done) { - if (settings.limit > 0) + if (settings[Setting::limit] > 0) { - auto limit = std::make_unique(query_plan.getCurrentDataStream(), settings.limit, settings.offset, settings.exact_rows_before_limit); + auto limit = std::make_unique( + query_plan.getCurrentDataStream(), settings[Setting::limit], settings[Setting::offset], settings[Setting::exact_rows_before_limit]); limit->setStepDescription("LIMIT OFFSET for SETTINGS"); query_plan.addStep(std::move(limit)); } else { - auto offset = std::make_unique(query_plan.getCurrentDataStream(), settings.offset); + auto offset = std::make_unique(query_plan.getCurrentDataStream(), settings[Setting::offset]); offset->setStepDescription("OFFSET for SETTINGS"); query_plan.addStep(std::move(offset)); } diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp index 472cdedf3ae..49ccd96600d 100644 --- a/src/Interpreters/InterpreterShowColumnsQuery.cpp +++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool mysql_map_fixed_string_to_text_in_show_columns; + extern const SettingsBool mysql_map_string_to_text_in_show_columns; +} InterpreterShowColumnsQuery::InterpreterShowColumnsQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) @@ -32,8 +37,8 @@ String InterpreterShowColumnsQuery::getRewrittenQuery() const bool use_mysql_types = (client_interface == ClientInfo::Interface::MYSQL); // connection made through MySQL wire protocol const auto & settings = getContext()->getSettingsRef(); - const bool remap_string_as_text = settings.mysql_map_string_to_text_in_show_columns; - const bool remap_fixed_string_as_text = settings.mysql_map_fixed_string_to_text_in_show_columns; + const bool remap_string_as_text = settings[Setting::mysql_map_string_to_text_in_show_columns]; + const bool remap_fixed_string_as_text = settings[Setting::mysql_map_fixed_string_to_text_in_show_columns]; WriteBufferFromOwnString buf_database; String resolved_database = getContext()->resolveDatabase(query.database); diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp index 3de6b755609..bde75fd0ebf 100644 --- a/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -12,9 +12,14 @@ #include #include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsBool show_table_uuid_in_table_create_query_if_not_nil; +} namespace ErrorCodes { @@ -88,7 +93,7 @@ QueryPipeline InterpreterShowCreateQuery::executeImpl() "Unable to show the create query of {}. Maybe it was created by the system.", show_query->getTable()); - if (!getContext()->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil) + if (!getContext()->getSettingsRef()[Setting::show_table_uuid_in_table_create_query_if_not_nil]) { auto & create = create_query->as(); create.uuid = UUIDHelpers::Nil; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index d4e2f22036c..5018a2f118c 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -92,6 +92,11 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsSeconds receive_timeout; +} namespace ErrorCodes { @@ -870,7 +875,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, table->flushAndShutdown(); { /// If table was already dropped by anyone, an exception will be thrown - auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); + auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); create_ast = database->getCreateTableQuery(replica.table_name, getContext()); database->detachTable(system_context, replica.table_name); @@ -1135,7 +1140,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery & query) if (auto * storage_replicated = dynamic_cast(table.get())) { LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for current last entry to be processed"); - auto sync_timeout = getContext()->getSettingsRef().receive_timeout.totalMilliseconds(); + auto sync_timeout = getContext()->getSettingsRef()[Setting::receive_timeout].totalMilliseconds(); std::unordered_set replicas(query.src_replicas.begin(), query.src_replicas.end()); if (!storage_replicated->waitForProcessingQueue(sync_timeout, query.sync_replica_mode, replicas)) @@ -1213,7 +1218,7 @@ void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query) if (auto * ptr = typeid_cast(database.get())) { LOG_TRACE(log, "Synchronizing entries in the database replica's (name: {}) queue with the log", database_name); - if (!ptr->waitForReplicaToProcessAllEntries(getContext()->getSettingsRef().receive_timeout.totalMilliseconds())) + if (!ptr->waitForReplicaToProcessAllEntries(getContext()->getSettingsRef()[Setting::receive_timeout].totalMilliseconds())) { throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "SYNC DATABASE REPLICA {}: database is readonly or command timed out. " \ "See the 'receive_timeout' setting", database_name); diff --git a/src/Interpreters/InterpreterTransactionControlQuery.cpp b/src/Interpreters/InterpreterTransactionControlQuery.cpp index e92d6f9c5e7..217c8b9ff02 100644 --- a/src/Interpreters/InterpreterTransactionControlQuery.cpp +++ b/src/Interpreters/InterpreterTransactionControlQuery.cpp @@ -7,6 +7,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsTransactionsWaitCSNMode wait_changes_become_visible_after_commit_mode; +} namespace ErrorCodes { @@ -61,7 +65,7 @@ BlockIO InterpreterTransactionControlQuery::executeCommit(ContextMutablePtr sess if (txn->getState() != MergeTreeTransaction::RUNNING) throw Exception(ErrorCodes::INVALID_TRANSACTION, "Transaction is not in RUNNING state"); - TransactionsWaitCSNMode mode = query_context->getSettingsRef().wait_changes_become_visible_after_commit_mode; + TransactionsWaitCSNMode mode = query_context->getSettingsRef()[Setting::wait_changes_become_visible_after_commit_mode]; CSN csn; try { diff --git a/src/Interpreters/InterpreterWatchQuery.cpp b/src/Interpreters/InterpreterWatchQuery.cpp index 4937d8660e0..dae4b41f889 100644 --- a/src/Interpreters/InterpreterWatchQuery.cpp +++ b/src/Interpreters/InterpreterWatchQuery.cpp @@ -24,6 +24,17 @@ limitations under the License. */ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_live_view; + extern const SettingsBool allow_experimental_window_view; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_columns_to_read; + extern const SettingsUInt64 max_result_bytes; + extern const SettingsUInt64 max_result_rows; + extern const SettingsOverflowMode result_overflow_mode; +} + namespace ErrorCodes { @@ -44,9 +55,9 @@ BlockIO InterpreterWatchQuery::execute() StreamLocalLimits limits; limits.mode = LimitsMode::LIMITS_CURRENT; - limits.size_limits.max_rows = settings.max_result_rows; - limits.size_limits.max_bytes = settings.max_result_bytes; - limits.size_limits.overflow_mode = settings.result_overflow_mode; + limits.size_limits.max_rows = settings[Setting::max_result_rows]; + limits.size_limits.max_bytes = settings[Setting::max_result_bytes]; + limits.size_limits.overflow_mode = settings[Setting::result_overflow_mode]; res.pipeline.setLimitsAndQuota(limits, getContext()->getQuota()); } @@ -66,12 +77,10 @@ QueryPipelineBuilder InterpreterWatchQuery::buildQueryPipeline() throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} does not exist.", table_id.getNameForLogs()); auto storage_name = storage->getName(); - if (storage_name == "LiveView" - && !getContext()->getSettingsRef().allow_experimental_live_view) + if (storage_name == "LiveView" && !getContext()->getSettingsRef()[Setting::allow_experimental_live_view]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')"); - else if (storage_name == "WindowView" - && !getContext()->getSettingsRef().allow_experimental_window_view) + else if (storage_name == "WindowView" && !getContext()->getSettingsRef()[Setting::allow_experimental_window_view]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental WINDOW VIEW feature is not enabled (the setting 'allow_experimental_window_view')"); @@ -83,11 +92,15 @@ QueryPipelineBuilder InterpreterWatchQuery::buildQueryPipeline() const Settings & settings = getContext()->getSettingsRef(); /// Limitation on the number of columns to read. - if (settings.max_columns_to_read && required_columns.size() > settings.max_columns_to_read) - throw Exception(ErrorCodes::TOO_MANY_COLUMNS, "Limit for number of columns to read exceeded. " - "Requested: {}, maximum: {}", required_columns.size(), settings.max_columns_to_read.toString()); + if (settings[Setting::max_columns_to_read] && required_columns.size() > settings[Setting::max_columns_to_read]) + throw Exception( + ErrorCodes::TOO_MANY_COLUMNS, + "Limit for number of columns to read exceeded. " + "Requested: {}, maximum: {}", + required_columns.size(), + settings[Setting::max_columns_to_read].toString()); - size_t max_block_size = settings.max_block_size; + size_t max_block_size = settings[Setting::max_block_size]; size_t max_streams = 1; /// Define query info diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index c5226107f8d..e13c836d70e 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -28,6 +28,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool asterisk_include_alias_columns; + extern const SettingsBool asterisk_include_materialized_columns; + extern const SettingsBool enable_optimize_predicate_expression; + extern const SettingsBool joined_subquery_requires_alias; + extern const SettingsJoinAlgorithm join_algorithm; +} namespace ErrorCodes { @@ -240,13 +248,13 @@ StoragePtr JoinedTables::getLeftTableStorage() bool JoinedTables::resolveTables() { const auto & settings = context->getSettingsRef(); - bool include_alias_cols = include_all_columns || settings.asterisk_include_alias_columns; - bool include_materialized_cols = include_all_columns || settings.asterisk_include_materialized_columns; + bool include_alias_cols = include_all_columns || settings[Setting::asterisk_include_alias_columns]; + bool include_materialized_cols = include_all_columns || settings[Setting::asterisk_include_materialized_columns]; tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context, include_alias_cols, include_materialized_cols, is_create_parameterized_view); if (tables_with_columns.size() != table_expressions.size()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected tables count"); - if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) + if (settings[Setting::joined_subquery_requires_alias] && tables_with_columns.size() > 1) { for (size_t i = 0; i < tables_with_columns.size(); ++i) { @@ -309,7 +317,7 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se return {}; const auto & settings = context->getSettingsRef(); - MultiEnum join_algorithm = settings.join_algorithm; + MultiEnum join_algorithm = settings[Setting::join_algorithm]; bool try_use_direct_join = join_algorithm.isSet(JoinAlgorithm::DIRECT) || join_algorithm.isSet(JoinAlgorithm::DEFAULT); auto table_join = std::make_shared(settings, context->getGlobalTemporaryVolume(), context->getTempDataOnDisk()); @@ -357,8 +365,7 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se } } - if (!table_join->isSpecialStorage() && - settings.enable_optimize_predicate_expression) + if (!table_join->isSpecialStorage() && settings[Setting::enable_optimize_predicate_expression]) replaceJoinedTable(select_query_); return table_join; diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 916dee01431..1669691d802 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -47,6 +47,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_nondeterministic_mutations; + extern const SettingsUInt64 max_block_size; +} namespace ErrorCodes { @@ -189,7 +195,7 @@ bool isStorageTouchedByMutations( std::optional interpreter_select_query; BlockIO io; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { auto select_query_tree = prepareQueryAffectedQueryTree(commands, storage_from_part, context); InterpreterSelectQueryAnalyzer interpreter(select_query_tree, context, SelectQueryOptions().ignoreLimits()); @@ -421,7 +427,7 @@ MutationsInterpreter::MutationsInterpreter( , logger(getLogger("MutationsInterpreter(" + source.getStorage()->getStorageID().getFullTableName() + ")")) { auto new_context = Context::createCopy(context_); - if (new_context->getSettingsRef().allow_experimental_analyzer) + if (new_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { new_context->setSetting("allow_experimental_analyzer", false); LOG_DEBUG(logger, "Will use old analyzer to prepare mutation"); @@ -1270,7 +1276,7 @@ void MutationsInterpreter::Source::read( SelectQueryInfo query_info; query_info.query = std::move(select); - size_t max_block_size = context_->getSettingsRef().max_block_size; + size_t max_block_size = context_->getSettingsRef()[Setting::max_block_size]; size_t max_streams = 1; storage->read(plan, required_columns, storage_snapshot, query_info, context_, QueryProcessingStage::FetchColumns, max_block_size, max_streams); @@ -1340,7 +1346,7 @@ void MutationsInterpreter::validate() { /// For Replicated* storages mutations cannot employ non-deterministic functions /// because that produces inconsistencies between replicas - if (startsWith(source.getStorage()->getName(), "Replicated") && !context->getSettingsRef().allow_nondeterministic_mutations) + if (startsWith(source.getStorage()->getName(), "Replicated") && !context->getSettingsRef()[Setting::allow_nondeterministic_mutations]) { for (const auto & command : commands) { diff --git a/src/Interpreters/MutationsNonDeterministicHelpers.cpp b/src/Interpreters/MutationsNonDeterministicHelpers.cpp index bcff3e18b25..a220249058e 100644 --- a/src/Interpreters/MutationsNonDeterministicHelpers.cpp +++ b/src/Interpreters/MutationsNonDeterministicHelpers.cpp @@ -15,6 +15,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool mutations_execute_nondeterministic_on_initiator; + extern const SettingsBool mutations_execute_subqueries_on_initiator; + extern const SettingsUInt64 mutations_max_literal_size_to_replace; +} namespace { @@ -166,8 +172,7 @@ FirstNonDeterministicFunctionResult findFirstNonDeterministicFunction(const Muta ASTPtr replaceNonDeterministicToScalars(const ASTAlterCommand & alter_command, ContextPtr context) { const auto & settings = context->getSettingsRef(); - if (!settings.mutations_execute_subqueries_on_initiator - && !settings.mutations_execute_nondeterministic_on_initiator) + if (!settings[Setting::mutations_execute_subqueries_on_initiator] && !settings[Setting::mutations_execute_nondeterministic_on_initiator]) return nullptr; auto query = alter_command.clone(); @@ -197,29 +202,28 @@ ASTPtr replaceNonDeterministicToScalars(const ASTAlterCommand & alter_command, C } }; - if (settings.mutations_execute_subqueries_on_initiator) + if (settings[Setting::mutations_execute_subqueries_on_initiator]) { Scalars scalars; Scalars local_scalars; ExecuteScalarSubqueriesVisitor::Data data{ WithContext{context}, - /*subquery_depth=*/ 0, + /*subquery_depth=*/0, scalars, local_scalars, - /*only_analyze=*/ false, - /*is_create_parameterized_view=*/ false, - /*replace_only_to_literals=*/ true, - settings.mutations_max_literal_size_to_replace}; + /*only_analyze=*/false, + /*is_create_parameterized_view=*/false, + /*replace_only_to_literals=*/true, + settings[Setting::mutations_max_literal_size_to_replace]}; ExecuteScalarSubqueriesVisitor visitor(data); visit(visitor); } - if (settings.mutations_execute_nondeterministic_on_initiator) + if (settings[Setting::mutations_execute_nondeterministic_on_initiator]) { - ExecuteNonDeterministicConstFunctionsVisitor::Data data{ - context, settings.mutations_max_literal_size_to_replace}; + ExecuteNonDeterministicConstFunctionsVisitor::Data data{context, settings[Setting::mutations_max_literal_size_to_replace]}; ExecuteNonDeterministicConstFunctionsVisitor visitor(data); visit(visitor); diff --git a/src/Interpreters/PredicateExpressionsOptimizer.cpp b/src/Interpreters/PredicateExpressionsOptimizer.cpp index af67e62be1a..580097d949f 100644 --- a/src/Interpreters/PredicateExpressionsOptimizer.cpp +++ b/src/Interpreters/PredicateExpressionsOptimizer.cpp @@ -15,6 +15,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_push_predicate_when_subquery_contains_with; + extern const SettingsBool enable_optimize_predicate_expression; + extern const SettingsBool enable_optimize_predicate_expression_to_final_subquery; +} namespace ErrorCodes { @@ -24,9 +30,9 @@ namespace ErrorCodes PredicateExpressionsOptimizer::PredicateExpressionsOptimizer( ContextPtr context_, const TablesWithColumns & tables_with_columns_, const Settings & settings) : WithContext(context_) - , enable_optimize_predicate_expression(settings.enable_optimize_predicate_expression) - , enable_optimize_predicate_expression_to_final_subquery(settings.enable_optimize_predicate_expression_to_final_subquery) - , allow_push_predicate_when_subquery_contains_with(settings.allow_push_predicate_when_subquery_contains_with) + , enable_optimize_predicate_expression(settings[Setting::enable_optimize_predicate_expression]) + , enable_optimize_predicate_expression_to_final_subquery(settings[Setting::enable_optimize_predicate_expression_to_final_subquery]) + , allow_push_predicate_when_subquery_contains_with(settings[Setting::allow_push_predicate_when_subquery_contains_with]) , tables_with_columns(tables_with_columns_) { } diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 43a04f8aff0..3e37b2191a0 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -15,11 +15,22 @@ #include #include #include -#include #include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_bytes_in_set; + extern const SettingsUInt64 max_bytes_to_transfer; + extern const SettingsUInt64 max_rows_in_set; + extern const SettingsUInt64 max_rows_to_transfer; + extern const SettingsOverflowMode set_overflow_mode; + extern const SettingsOverflowMode transfer_overflow_mode; + extern const SettingsBool transform_null_in; + extern const SettingsBool use_index_for_in_with_subqueries; + extern const SettingsUInt64 use_index_for_in_with_subqueries_max_values; +} namespace ErrorCodes { @@ -28,7 +39,7 @@ namespace ErrorCodes static SizeLimits getSizeLimitsForSet(const Settings & settings) { - return SizeLimits(settings.max_rows_in_set, settings.max_bytes_in_set, settings.set_overflow_mode); + return SizeLimits(settings[Setting::max_rows_in_set], settings[Setting::max_bytes_in_set], settings[Setting::set_overflow_mode]); } static bool equals(const DataTypes & lhs, const DataTypes & rhs) @@ -60,7 +71,7 @@ SetPtr FutureSetFromStorage::buildOrderedSetInplace(const ContextPtr &) FutureSetFromTuple::FutureSetFromTuple(Block block, const Settings & settings) { auto size_limits = getSizeLimitsForSet(settings); - set = std::make_shared(size_limits, settings.use_index_for_in_with_subqueries_max_values, settings.transform_null_in); + set = std::make_shared(size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values], settings[Setting::transform_null_in]); set->setHeader(block.cloneEmpty().getColumnsWithTypeAndName()); Columns columns; @@ -82,7 +93,7 @@ SetPtr FutureSetFromTuple::buildOrderedSetInplace(const ContextPtr & context) return set; const auto & settings = context->getSettingsRef(); - size_t max_values = settings.use_index_for_in_with_subqueries_max_values; + size_t max_values = settings[Setting::use_index_for_in_with_subqueries_max_values]; bool too_many_values = max_values && max_values < set->getTotalRowCount(); if (!too_many_values) { @@ -106,7 +117,8 @@ FutureSetFromSubquery::FutureSetFromSubquery( set_and_key->key = std::move(key); auto size_limits = getSizeLimitsForSet(settings); - set_and_key->set = std::make_shared(size_limits, settings.use_index_for_in_with_subqueries_max_values, settings.transform_null_in); + set_and_key->set + = std::make_shared(size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values], settings[Setting::transform_null_in]); set_and_key->set->setHeader(source->getCurrentDataStream().header.getColumnsWithTypeAndName()); } @@ -120,7 +132,8 @@ FutureSetFromSubquery::FutureSetFromSubquery( set_and_key->key = std::move(key); auto size_limits = getSizeLimitsForSet(settings); - set_and_key->set = std::make_shared(size_limits, settings.use_index_for_in_with_subqueries_max_values, settings.transform_null_in); + set_and_key->set + = std::make_shared(size_limits, settings[Setting::use_index_for_in_with_subqueries_max_values], settings[Setting::transform_null_in]); } FutureSetFromSubquery::~FutureSetFromSubquery() = default; @@ -157,11 +170,11 @@ std::unique_ptr FutureSetFromSubquery::build(const ContextPtr & conte return nullptr; auto creating_set = std::make_unique( - plan->getCurrentDataStream(), - set_and_key, - external_table, - SizeLimits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode), - context); + plan->getCurrentDataStream(), + set_and_key, + external_table, + SizeLimits(settings[Setting::max_rows_to_transfer], settings[Setting::max_bytes_to_transfer], settings[Setting::transfer_overflow_mode]), + context); creating_set->setStepDescription("Create set for subquery"); plan->addStep(std::move(creating_set)); return plan; @@ -187,7 +200,7 @@ void FutureSetFromSubquery::buildSetInplace(const ContextPtr & context) SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) { - if (!context->getSettingsRef().use_index_for_in_with_subqueries) + if (!context->getSettingsRef()[Setting::use_index_for_in_with_subqueries]) return nullptr; if (auto set = get()) diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index f8a808f6c68..402cbee3c7c 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -21,6 +21,33 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_concurrent_queries_for_all_users; + extern const SettingsUInt64 max_concurrent_queries_for_user; + extern const SettingsSeconds max_execution_time; + extern const SettingsUInt64 max_memory_usage; + extern const SettingsUInt64 max_memory_usage_for_user; + extern const SettingsUInt64 max_network_bandwidth_for_all_users; + extern const SettingsUInt64 max_network_bandwidth_for_user; + extern const SettingsUInt64 max_temporary_data_on_disk_size_for_user; + extern const SettingsUInt64 memory_usage_overcommit_max_wait_microseconds; + extern const SettingsUInt64 memory_overcommit_ratio_denominator; + extern const SettingsUInt64 memory_overcommit_ratio_denominator_for_user; + extern const SettingsUInt64 memory_profiler_step; + extern const SettingsUInt64 memory_profiler_sample_min_allocation_size; + extern const SettingsUInt64 memory_profiler_sample_max_allocation_size; + extern const SettingsFloat memory_profiler_sample_probability; + extern const SettingsUInt64 max_temporary_data_on_disk_size_for_query; + extern const SettingsFloat memory_tracker_fault_probability; + extern const SettingsUInt64 priority; + extern const SettingsMilliseconds queue_max_wait_ms; + extern const SettingsBool replace_running_query; + extern const SettingsMilliseconds replace_running_query_max_wait_ms; + extern const SettingsString temporary_files_codec; + extern const SettingsOverflowMode timeout_overflow_mode; + extern const SettingsBool trace_profile_events; +} namespace ErrorCodes { @@ -82,7 +109,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q auto [lock, overcommit_blocker] = safeLock(); // To avoid deadlock in case of OOM IAST::QueryKind query_kind = ast->getQueryKind(); - const auto queue_max_wait_ms = settings.queue_max_wait_ms.totalMilliseconds(); + const auto queue_max_wait_ms = settings[Setting::queue_max_wait_ms].totalMilliseconds(); UInt64 waiting_queries = waiting_queries_amount.load(); if (!is_unlimited_query && max_size && processes.size() >= max_size + waiting_queries) { @@ -131,10 +158,14 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q */ waiting_queries = waiting_queries_amount.load(); - if (!is_unlimited_query && settings.max_concurrent_queries_for_all_users - && processes.size() >= settings.max_concurrent_queries_for_all_users + waiting_queries_amount) - throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, "Too many simultaneous queries for all users. " - "Current: {}, maximum: {}{}", processes.size(), settings.max_concurrent_queries_for_all_users.toString(), + if (!is_unlimited_query && settings[Setting::max_concurrent_queries_for_all_users] + && processes.size() >= settings[Setting::max_concurrent_queries_for_all_users] + waiting_queries_amount) + throw Exception( + ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, + "Too many simultaneous queries for all users. " + "Current: {}, maximum: {}{}", + processes.size(), + settings[Setting::max_concurrent_queries_for_all_users].toString(), waiting_queries == 0 ? "" : fmt::format(", waiting: {}", waiting_queries)); } @@ -154,26 +185,28 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q if (user_process_list != user_to_queries.end()) { UInt64 user_waiting_queries = user_process_list->second.waiting_queries_amount.load(); - if (!is_unlimited_query && settings.max_concurrent_queries_for_user - && user_process_list->second.queries.size() >= settings.max_concurrent_queries_for_user + user_waiting_queries) - throw Exception(ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, - "Too many simultaneous queries for user {}. " - "Current: {}, maximum: {}{}", - client_info.current_user, user_process_list->second.queries.size(), - settings.max_concurrent_queries_for_user.toString(), - user_waiting_queries == 0 ? "" : fmt::format(", waiting: {}", user_waiting_queries)); + if (!is_unlimited_query && settings[Setting::max_concurrent_queries_for_user] + && user_process_list->second.queries.size() >= settings[Setting::max_concurrent_queries_for_user] + user_waiting_queries) + throw Exception( + ErrorCodes::TOO_MANY_SIMULTANEOUS_QUERIES, + "Too many simultaneous queries for user {}. " + "Current: {}, maximum: {}{}", + client_info.current_user, + user_process_list->second.queries.size(), + settings[Setting::max_concurrent_queries_for_user].toString(), + user_waiting_queries == 0 ? "" : fmt::format(", waiting: {}", user_waiting_queries)); auto running_query = user_process_list->second.queries.find(client_info.current_query_id); if (running_query != user_process_list->second.queries.end()) { - if (!settings.replace_running_query) + if (!settings[Setting::replace_running_query]) throw Exception(ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING, "Query with id = {} is already running.", client_info.current_query_id); /// Ask queries to cancel. They will check this flag. running_query->second->is_killed.store(true, std::memory_order_relaxed); - const auto replace_running_query_max_wait_ms = settings.replace_running_query_max_wait_ms.totalMilliseconds(); + const auto replace_running_query_max_wait_ms = settings[Setting::replace_running_query_max_wait_ms].totalMilliseconds(); if (!replace_running_query_max_wait_ms || !have_space.wait_for(lock, std::chrono::milliseconds(replace_running_query_max_wait_ms), [&] { @@ -188,7 +221,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q "Query with id = {} is already running and can't be stopped", client_info.current_query_id); } - } + } } } @@ -220,33 +253,33 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q { TemporaryDataOnDiskSettings temporary_data_on_disk_settings { - .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_query, - .compression_codec = settings.temporary_files_codec + .max_size_on_disk = settings[Setting::max_temporary_data_on_disk_size_for_query], + .compression_codec = settings[Setting::temporary_files_codec] }; query_context->setTempDataOnDisk(std::make_shared( user_process_list.user_temp_data_on_disk, std::move(temporary_data_on_disk_settings))); } /// Set query-level memory trackers - thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage); - thread_group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator); + thread_group->memory_tracker.setOrRaiseHardLimit(settings[Setting::max_memory_usage]); + thread_group->memory_tracker.setSoftLimit(settings[Setting::memory_overcommit_ratio_denominator]); if (query_context->hasTraceCollector()) { /// Set up memory profiling - thread_group->memory_tracker.setProfilerStep(settings.memory_profiler_step); + thread_group->memory_tracker.setProfilerStep(settings[Setting::memory_profiler_step]); - thread_group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); - thread_group->memory_tracker.setSampleMinAllocationSize(settings.memory_profiler_sample_min_allocation_size); - thread_group->memory_tracker.setSampleMaxAllocationSize(settings.memory_profiler_sample_max_allocation_size); - thread_group->performance_counters.setTraceProfileEvents(settings.trace_profile_events); + thread_group->memory_tracker.setSampleProbability(settings[Setting::memory_profiler_sample_probability]); + thread_group->memory_tracker.setSampleMinAllocationSize(settings[Setting::memory_profiler_sample_min_allocation_size]); + thread_group->memory_tracker.setSampleMaxAllocationSize(settings[Setting::memory_profiler_sample_max_allocation_size]); + thread_group->performance_counters.setTraceProfileEvents(settings[Setting::trace_profile_events]); } thread_group->memory_tracker.setDescription("(for query)"); - if (settings.memory_tracker_fault_probability > 0.0) - thread_group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); + if (settings[Setting::memory_tracker_fault_probability] > 0.0) + thread_group->memory_tracker.setFaultProbability(settings[Setting::memory_tracker_fault_probability]); - thread_group->memory_tracker.setOvercommitWaitingTime(settings.memory_usage_overcommit_max_wait_microseconds); + thread_group->memory_tracker.setOvercommitWaitingTime(settings[Setting::memory_usage_overcommit_max_wait_microseconds]); /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values /// since allocation and deallocation could happen in different threads @@ -258,7 +291,7 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q query_context, query_, client_info, - priorities.insert(settings.priority), + priorities.insert(settings[Setting::priority]), std::move(thread_group), query_kind, settings, @@ -275,20 +308,21 @@ ProcessList::insert(const String & query_, const IAST * ast, ContextMutablePtr q queries_to_user.emplace(client_info.current_query_id, client_info.current_user); /// Track memory usage for all simultaneously running queries from single user. - user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); - user_process_list.user_memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator_for_user); + user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings[Setting::max_memory_usage_for_user]); + user_process_list.user_memory_tracker.setSoftLimit(settings[Setting::memory_overcommit_ratio_denominator_for_user]); user_process_list.user_memory_tracker.setDescription("(for user)"); - if (!total_network_throttler && settings.max_network_bandwidth_for_all_users) + if (!total_network_throttler && settings[Setting::max_network_bandwidth_for_all_users]) { - total_network_throttler = std::make_shared(settings.max_network_bandwidth_for_all_users); + total_network_throttler = std::make_shared(settings[Setting::max_network_bandwidth_for_all_users]); } if (!user_process_list.user_throttler) { - if (settings.max_network_bandwidth_for_user) - user_process_list.user_throttler = std::make_shared(settings.max_network_bandwidth_for_user, total_network_throttler); - else if (settings.max_network_bandwidth_for_all_users) + if (settings[Setting::max_network_bandwidth_for_user]) + user_process_list.user_throttler + = std::make_shared(settings[Setting::max_network_bandwidth_for_user], total_network_throttler); + else if (settings[Setting::max_network_bandwidth_for_all_users]) user_process_list.user_throttler = total_network_throttler; } } @@ -378,8 +412,8 @@ QueryStatus::QueryStatus( /// We have to pass `query_settings_` to this constructor because we can't use `context_->getSettings().max_execution_time` here: /// a QueryStatus is created with `ProcessList::mutex` locked (see ProcessList::insert) and calling `context_->getSettings()` /// would lock the context's lock too, whereas holding two those locks simultaneously is not good. - limits.max_execution_time = query_settings_.max_execution_time; - overflow_mode = query_settings_.timeout_overflow_mode; + limits.max_execution_time = query_settings_[Setting::max_execution_time]; + overflow_mode = query_settings_[Setting::timeout_overflow_mode]; } QueryStatus::~QueryStatus() @@ -700,8 +734,8 @@ ProcessListForUser::ProcessListForUser(ContextPtr global_context, ProcessList * const auto & settings = global_context->getSettingsRef(); TemporaryDataOnDiskSettings temporary_data_on_disk_settings { - .max_size_on_disk = settings.max_temporary_data_on_disk_size_for_user, - .compression_codec = settings.temporary_files_codec + .max_size_on_disk = settings[Setting::max_temporary_data_on_disk_size_for_user], + .compression_codec = settings[Setting::temporary_files_codec] }; user_temp_data_on_disk = std::make_shared(global_context->getSharedTempDataOnDisk(), diff --git a/src/Interpreters/QueryNormalizer.h b/src/Interpreters/QueryNormalizer.h index 90c70dd71e6..b0386a3477d 100644 --- a/src/Interpreters/QueryNormalizer.h +++ b/src/Interpreters/QueryNormalizer.h @@ -2,9 +2,10 @@ #include -#include -#include #include +#include +#include +#include namespace DB { @@ -15,6 +16,13 @@ struct ASTTablesInSelectQueryElement; class Context; class ASTQueryParameter; +namespace Setting +{ + extern const SettingsUInt64 max_ast_depth; + extern const SettingsUInt64 max_expanded_ast_elements; + extern const SettingsBool prefer_column_name_to_alias; +} + class QueryNormalizer { @@ -27,9 +35,9 @@ class QueryNormalizer template ExtractedSettings(const T & settings) /// NOLINT - : max_ast_depth(settings.max_ast_depth) - , max_expanded_ast_elements(settings.max_expanded_ast_elements) - , prefer_column_name_to_alias(settings.prefer_column_name_to_alias) + : max_ast_depth(settings[Setting::max_ast_depth]) + , max_expanded_ast_elements(settings[Setting::max_expanded_ast_elements]) + , prefer_column_name_to_alias(settings[Setting::prefer_column_name_to_alias]) { } }; diff --git a/src/Interpreters/RewriteCountVariantsVisitor.cpp b/src/Interpreters/RewriteCountVariantsVisitor.cpp index 272e1ac735f..b4867649a5b 100644 --- a/src/Interpreters/RewriteCountVariantsVisitor.cpp +++ b/src/Interpreters/RewriteCountVariantsVisitor.cpp @@ -12,6 +12,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool aggregate_functions_null_for_empty; +} void RewriteCountVariantsVisitor::visit(ASTPtr & node) { @@ -54,7 +58,7 @@ void RewriteCountVariantsVisitor::visit(ASTFunction & func) if (first_arg_literal->value.getType() == Field::Types::UInt64) { auto constant = first_arg_literal->value.safeGet(); - if (constant == 1 && !context->getSettingsRef().aggregate_functions_null_for_empty) + if (constant == 1 && !context->getSettingsRef()[Setting::aggregate_functions_null_for_empty]) transform = true; } } diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index a82e8d76c4e..ede91b0df56 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -28,6 +28,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_sessions_for_user; +} namespace ErrorCodes { @@ -538,7 +542,7 @@ ContextMutablePtr Session::makeSessionContext() session_tracker_handle = session_context->getSessionTracker().trackSession( *user_id, {}, - session_context->getSettingsRef().max_sessions_for_user); + session_context->getSettingsRef()[Setting::max_sessions_for_user]); // Use QUERY source as for SET query for a session session_context->checkSettingsConstraints(settings_from_auth_server, SettingSource::QUERY); @@ -585,7 +589,7 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: if (!access->tryGetUser()) { new_session_context->setUser(*user_id); - max_sessions_for_user = new_session_context->getSettingsRef().max_sessions_for_user; + max_sessions_for_user = new_session_context->getSettingsRef()[Setting::max_sessions_for_user]; } else { diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index ef102f94308..afbd4ed45a1 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -237,8 +237,9 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, if (const auto profile_info = access->getDefaultProfileInfo()) log_entry.profiles = profile_info->getProfileNames(); - for (const auto & s : settings.allChanged()) - log_entry.settings.emplace_back(s.getName(), s.getValueString()); + SettingsChanges changes = settings.changes(); + for (const auto & change : changes) + log_entry.settings.emplace_back(change.name, Settings::valueToStringUtil(change.name, change.value)); add(std::move(log_entry)); } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 59a0374051f..5aa06db8e58 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -38,6 +38,27 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_join_right_table_sorting; + extern const SettingsUInt64 cross_join_min_bytes_to_compress; + extern const SettingsUInt64 cross_join_min_rows_to_compress; + extern const SettingsUInt64 default_max_bytes_in_join; + extern const SettingsJoinAlgorithm join_algorithm; + extern const SettingsUInt64 join_on_disk_max_files_to_merge; + extern const SettingsInt32 join_output_by_rowlist_perkey_rows_threshold; + extern const SettingsOverflowMode join_overflow_mode; + extern const SettingsInt32 join_to_sort_maximum_table_rows; + extern const SettingsInt32 join_to_sort_minimum_perkey_rows; + extern const SettingsBool join_use_nulls; + extern const SettingsUInt64 max_bytes_in_join; + extern const SettingsUInt64 max_joined_block_size_rows; + extern const SettingsUInt64 max_memory_usage; + extern const SettingsUInt64 max_rows_in_join; + extern const SettingsUInt64 partial_merge_join_left_table_buffer_bytes; + extern const SettingsUInt64 partial_merge_join_rows_in_right_blocks; + extern const SettingsString temporary_files_codec; +} namespace ErrorCodes { @@ -104,22 +125,22 @@ bool forAllKeys(OnExpr & expressions, Func callback) } TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, TemporaryDataOnDiskScopePtr tmp_data_) - : size_limits(SizeLimits{settings.max_rows_in_join, settings.max_bytes_in_join, settings.join_overflow_mode}) - , default_max_bytes(settings.default_max_bytes_in_join) - , join_use_nulls(settings.join_use_nulls) - , cross_join_min_rows_to_compress(settings.cross_join_min_rows_to_compress) - , cross_join_min_bytes_to_compress(settings.cross_join_min_bytes_to_compress) - , max_joined_block_rows(settings.max_joined_block_size_rows) - , join_algorithm(settings.join_algorithm) - , partial_merge_join_rows_in_right_blocks(settings.partial_merge_join_rows_in_right_blocks) - , partial_merge_join_left_table_buffer_bytes(settings.partial_merge_join_left_table_buffer_bytes) - , max_files_to_merge(settings.join_on_disk_max_files_to_merge) - , temporary_files_codec(settings.temporary_files_codec) - , output_by_rowlist_perkey_rows_threshold(settings.join_output_by_rowlist_perkey_rows_threshold) - , sort_right_minimum_perkey_rows(settings.join_to_sort_minimum_perkey_rows) - , sort_right_maximum_table_rows(settings.join_to_sort_maximum_table_rows) - , allow_join_sorting(settings.allow_experimental_join_right_table_sorting) - , max_memory_usage(settings.max_memory_usage) + : size_limits(SizeLimits{settings[Setting::max_rows_in_join], settings[Setting::max_bytes_in_join], settings[Setting::join_overflow_mode]}) + , default_max_bytes(settings[Setting::default_max_bytes_in_join]) + , join_use_nulls(settings[Setting::join_use_nulls]) + , cross_join_min_rows_to_compress(settings[Setting::cross_join_min_rows_to_compress]) + , cross_join_min_bytes_to_compress(settings[Setting::cross_join_min_bytes_to_compress]) + , max_joined_block_rows(settings[Setting::max_joined_block_size_rows]) + , join_algorithm(settings[Setting::join_algorithm]) + , partial_merge_join_rows_in_right_blocks(settings[Setting::partial_merge_join_rows_in_right_blocks]) + , partial_merge_join_left_table_buffer_bytes(settings[Setting::partial_merge_join_left_table_buffer_bytes]) + , max_files_to_merge(settings[Setting::join_on_disk_max_files_to_merge]) + , temporary_files_codec(settings[Setting::temporary_files_codec]) + , output_by_rowlist_perkey_rows_threshold(settings[Setting::join_output_by_rowlist_perkey_rows_threshold]) + , sort_right_minimum_perkey_rows(settings[Setting::join_to_sort_minimum_perkey_rows]) + , sort_right_maximum_table_rows(settings[Setting::join_to_sort_maximum_table_rows]) + , allow_join_sorting(settings[Setting::allow_experimental_join_right_table_sorting]) + , max_memory_usage(settings[Setting::max_memory_usage]) , tmp_volume(tmp_volume_) , tmp_data(tmp_data_) { diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index ca8f8d235fa..0544bbcc92e 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -37,6 +37,28 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool calculate_text_stack_trace; + extern const SettingsBool enable_job_stack_trace; + extern const SettingsBool log_queries; + extern const SettingsMilliseconds log_queries_min_query_duration_ms; + extern const SettingsBool log_profile_events; + extern const SettingsUInt64 log_queries_cut_to_length; + extern const SettingsBool log_query_threads; + extern const SettingsUInt64 max_untracked_memory; + extern const SettingsUInt64 memory_overcommit_ratio_denominator; + extern const SettingsFloat memory_profiler_sample_probability; + extern const SettingsUInt64 memory_profiler_sample_min_allocation_size; + extern const SettingsUInt64 memory_profiler_sample_max_allocation_size; + extern const SettingsUInt64 memory_profiler_step; + extern const SettingsFloat memory_tracker_fault_probability; + extern const SettingsBool metrics_perf_events_enabled; + extern const SettingsString metrics_perf_events_list; + extern const SettingsInt64 os_thread_priority; + extern const SettingsUInt64 query_profiler_cpu_time_period_ns; + extern const SettingsUInt64 query_profiler_real_time_period_ns; +} namespace ErrorCodes { @@ -108,14 +130,14 @@ ThreadGroupPtr ThreadGroup::createForBackgroundProcess(ContextPtr storage_contex group->memory_tracker.setDescription("background process to apply mutate/merge in table"); /// However settings from storage context have to be applied const Settings & settings = storage_context->getSettingsRef(); - group->memory_tracker.setProfilerStep(settings.memory_profiler_step); - group->memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability); - group->memory_tracker.setSampleMinAllocationSize(settings.memory_profiler_sample_min_allocation_size); - group->memory_tracker.setSampleMaxAllocationSize(settings.memory_profiler_sample_max_allocation_size); - group->memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator); + group->memory_tracker.setProfilerStep(settings[Setting::memory_profiler_step]); + group->memory_tracker.setSampleProbability(settings[Setting::memory_profiler_sample_probability]); + group->memory_tracker.setSampleMinAllocationSize(settings[Setting::memory_profiler_sample_min_allocation_size]); + group->memory_tracker.setSampleMaxAllocationSize(settings[Setting::memory_profiler_sample_max_allocation_size]); + group->memory_tracker.setSoftLimit(settings[Setting::memory_overcommit_ratio_denominator]); group->memory_tracker.setParent(&background_memory_tracker); - if (settings.memory_tracker_fault_probability > 0.0) - group->memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability); + if (settings[Setting::memory_tracker_fault_probability] > 0.0) + group->memory_tracker.setFaultProbability(settings[Setting::memory_tracker_fault_probability]); return group; } @@ -195,7 +217,7 @@ void ThreadStatus::applyGlobalSettings() const Settings & settings = global_context_ptr->getSettingsRef(); - DB::Exception::enable_job_stack_trace = settings.enable_job_stack_trace; + DB::Exception::enable_job_stack_trace = settings[Setting::enable_job_stack_trace]; } void ThreadStatus::applyQuerySettings() @@ -206,18 +228,18 @@ void ThreadStatus::applyQuerySettings() const Settings & settings = query_context_ptr->getSettingsRef(); - DB::Exception::enable_job_stack_trace = settings.enable_job_stack_trace; + DB::Exception::enable_job_stack_trace = settings[Setting::enable_job_stack_trace]; query_id_from_query_context = query_context_ptr->getCurrentQueryId(); initQueryProfiler(); - untracked_memory_limit = settings.max_untracked_memory; - if (settings.memory_profiler_step && settings.memory_profiler_step < static_cast(untracked_memory_limit)) - untracked_memory_limit = settings.memory_profiler_step; + untracked_memory_limit = settings[Setting::max_untracked_memory]; + if (settings[Setting::memory_profiler_step] && settings[Setting::memory_profiler_step] < static_cast(untracked_memory_limit)) + untracked_memory_limit = settings[Setting::memory_profiler_step]; #if defined(OS_LINUX) /// Set "nice" value if required. - Int32 new_os_thread_priority = static_cast(settings.os_thread_priority); + Int32 new_os_thread_priority = static_cast(settings[Setting::os_thread_priority]); if (new_os_thread_priority && hasLinuxCapability(CAP_SYS_NICE)) { LOG_TRACE(log, "Setting nice to {}", new_os_thread_priority); @@ -374,12 +396,11 @@ void ThreadStatus::initPerformanceCounters() if (auto query_context_ptr = query_context.lock()) { const Settings & settings = query_context_ptr->getSettingsRef(); - if (settings.metrics_perf_events_enabled) + if (settings[Setting::metrics_perf_events_enabled]) { try { - current_thread_counters.initializeProfileEvents( - settings.metrics_perf_events_list); + current_thread_counters.initializeProfileEvents(settings[Setting::metrics_perf_events_list]); } catch (...) { @@ -416,7 +437,7 @@ void ThreadStatus::finalizePerformanceCounters() // one query. bool close_perf_descriptors = true; if (auto global_context_ptr = global_context.lock()) - close_perf_descriptors = !global_context_ptr->getSettingsRef().metrics_perf_events_enabled; + close_perf_descriptors = !global_context_ptr->getSettingsRef()[Setting::metrics_perf_events_enabled]; try { @@ -436,11 +457,11 @@ void ThreadStatus::finalizePerformanceCounters() if (global_context_ptr && query_context_ptr) { const auto & settings = query_context_ptr->getSettingsRef(); - if (settings.log_queries && settings.log_query_threads) + if (settings[Setting::log_queries] && settings[Setting::log_query_threads]) { const auto now = std::chrono::system_clock::now(); Int64 query_duration_ms = std::chrono::duration_cast(now - query_start_time.point).count(); - if (query_duration_ms >= settings.log_queries_min_query_duration_ms.totalMilliseconds()) + if (query_duration_ms >= settings[Setting::log_queries_min_query_duration_ms].totalMilliseconds()) { if (auto thread_log = global_context_ptr->getQueryThreadLog()) logToQueryThreadLog(*thread_log, query_context_ptr->getCurrentDatabase()); @@ -502,22 +523,24 @@ void ThreadStatus::initQueryProfiler() try { - if (settings.query_profiler_real_time_period_ns > 0) + if (settings[Setting::query_profiler_real_time_period_ns] > 0) { if (!query_profiler_real) - query_profiler_real = std::make_unique(thread_id, - /* period= */ settings.query_profiler_real_time_period_ns); + query_profiler_real = std::make_unique( + thread_id, + /* period= */ settings[Setting::query_profiler_real_time_period_ns]); else - query_profiler_real->setPeriod(settings.query_profiler_real_time_period_ns); + query_profiler_real->setPeriod(settings[Setting::query_profiler_real_time_period_ns]); } - if (settings.query_profiler_cpu_time_period_ns > 0) + if (settings[Setting::query_profiler_cpu_time_period_ns] > 0) { if (!query_profiler_cpu) - query_profiler_cpu = std::make_unique(thread_id, - /* period= */ settings.query_profiler_cpu_time_period_ns); + query_profiler_cpu = std::make_unique( + thread_id, + /* period= */ settings[Setting::query_profiler_cpu_time_period_ns]); else - query_profiler_cpu->setPeriod(settings.query_profiler_cpu_time_period_ns); + query_profiler_cpu->setPeriod(settings[Setting::query_profiler_cpu_time_period_ns]); } } catch (...) @@ -572,7 +595,7 @@ void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log, const String { elem.client_info = query_context_ptr->getClientInfo(); - if (query_context_ptr->getSettingsRef().log_profile_events != 0) + if (query_context_ptr->getSettingsRef()[Setting::log_profile_events] != 0) { /// NOTE: Here we are in the same thread, so we can make memcpy() elem.profile_counters = std::make_shared(performance_counters.getPartiallyAtomicSnapshot()); @@ -588,7 +611,7 @@ static String getCleanQueryAst(const ASTPtr q, ContextPtr context) if (auto masker = SensitiveDataMasker::getInstance()) masker->wipeSensitiveData(res); - res = res.substr(0, context->getSettingsRef().log_queries_cut_to_length); + res = res.substr(0, context->getSettingsRef()[Setting::log_queries_cut_to_length]); return res; } @@ -625,7 +648,7 @@ void ThreadStatus::logToQueryViewsLog(const ViewRuntimeData & vinfo) element.written_rows = progress_out.written_rows.load(std::memory_order_relaxed); element.written_bytes = progress_out.written_bytes.load(std::memory_order_relaxed); element.peak_memory_usage = memory_tracker.getPeak() > 0 ? memory_tracker.getPeak() : 0; - if (query_context_ptr->getSettingsRef().log_profile_events != 0) + if (query_context_ptr->getSettingsRef()[Setting::log_profile_events] != 0) element.profile_counters = std::make_shared( performance_counters.getPartiallyAtomicSnapshot()); @@ -635,7 +658,7 @@ void ThreadStatus::logToQueryViewsLog(const ViewRuntimeData & vinfo) { element.exception_code = getExceptionErrorCode(vinfo.exception); element.exception = getExceptionMessage(vinfo.exception, false); - if (query_context_ptr->getSettingsRef().calculate_text_stack_trace) + if (query_context_ptr->getSettingsRef()[Setting::calculate_text_stack_trace]) element.stack_trace = getExceptionStackTraceString(vinfo.exception); } diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 6483dd3be48..7f794c3e90e 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -40,6 +40,27 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_hyperscan; + extern const SettingsBool convert_query_to_cnf; + extern const SettingsBool enable_positional_arguments; + extern const SettingsUInt64 max_hyperscan_regexp_length; + extern const SettingsUInt64 max_hyperscan_regexp_total_length; + extern const SettingsBool optimize_aggregators_of_group_by_keys; + extern const SettingsBool optimize_append_index; + extern const SettingsBool optimize_arithmetic_operations_in_aggregate_functions; + extern const SettingsBool optimize_group_by_function_keys; + extern const SettingsBool optimize_if_transform_strings_to_enum; + extern const SettingsBool optimize_injective_functions_inside_uniq; + extern const SettingsBool optimize_normalize_count_variants; + extern const SettingsBool optimize_substitute_columns; + extern const SettingsBool optimize_time_filter_with_preimage; + extern const SettingsBool optimize_using_constraints; + extern const SettingsBool optimize_redundant_functions_in_order_by; + extern const SettingsBool optimize_rewrite_array_exists_to_has; + extern const SettingsBool optimize_or_like_chain; +} namespace ErrorCodes { @@ -179,7 +200,7 @@ void optimizeGroupBy(ASTSelectQuery * select_query, ContextPtr context) else if (is_literal(group_exprs[i])) { bool keep_position = false; - if (settings.enable_positional_arguments) + if (settings[Setting::enable_positional_arguments]) { const auto & value = group_exprs[i]->as()->value; if (value.getType() == Field::Types::UInt64) @@ -629,57 +650,62 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, throw Exception(ErrorCodes::LOGICAL_ERROR, "Select analyze for not select asts."); /// Move arithmetic operations out of aggregation functions - if (settings.optimize_arithmetic_operations_in_aggregate_functions) + if (settings[Setting::optimize_arithmetic_operations_in_aggregate_functions]) optimizeAggregationFunctions(query); bool converted_to_cnf = false; - if (settings.convert_query_to_cnf) + if (settings[Setting::convert_query_to_cnf]) converted_to_cnf = convertQueryToCNF(select_query); - if (converted_to_cnf && settings.optimize_using_constraints && result.storage_snapshot) + if (converted_to_cnf && settings[Setting::optimize_using_constraints] && result.storage_snapshot) { - optimizeWithConstraints(select_query, result.aliases, result.source_columns_set, - tables_with_columns, result.storage_snapshot->metadata, settings.optimize_append_index); + optimizeWithConstraints( + select_query, + result.aliases, + result.source_columns_set, + tables_with_columns, + result.storage_snapshot->metadata, + settings[Setting::optimize_append_index]); - if (settings.optimize_substitute_columns) + if (settings[Setting::optimize_substitute_columns]) optimizeSubstituteColumn(select_query, result.aliases, result.source_columns_set, tables_with_columns, result.storage_snapshot->metadata, result.storage); } /// Rewrite sum(column +/- literal) function with sum(column) +/- literal * count(column). - if (settings.optimize_arithmetic_operations_in_aggregate_functions) + if (settings[Setting::optimize_arithmetic_operations_in_aggregate_functions]) rewriteSumFunctionWithSumAndCount(query, tables_with_columns); /// Rewrite date filters to avoid the calls of converters such as toYear, toYYYYMM, etc. - if (settings.optimize_time_filter_with_preimage) + if (settings[Setting::optimize_time_filter_with_preimage]) optimizeDateFilters(select_query, tables_with_columns, context); /// GROUP BY injective function elimination. optimizeGroupBy(select_query, context); /// GROUP BY functions of other keys elimination. - if (settings.optimize_group_by_function_keys) + if (settings[Setting::optimize_group_by_function_keys]) optimizeGroupByFunctionKeys(select_query); - if (settings.optimize_normalize_count_variants) + if (settings[Setting::optimize_normalize_count_variants]) optimizeCountConstantAndSumOne(query, context); - if (settings.optimize_rewrite_array_exists_to_has) + if (settings[Setting::optimize_rewrite_array_exists_to_has]) optimizeArrayExistsFunctions(query); /// Remove injective functions inside uniq - if (settings.optimize_injective_functions_inside_uniq) + if (settings[Setting::optimize_injective_functions_inside_uniq]) optimizeInjectiveFunctionsInsideUniq(query, context); /// Eliminate min/max/any aggregators of functions of GROUP BY keys - if (settings.optimize_aggregators_of_group_by_keys + if (settings[Setting::optimize_aggregators_of_group_by_keys] && !select_query->group_by_with_totals && !select_query->group_by_with_rollup && !select_query->group_by_with_cube) optimizeAggregateFunctionsOfGroupByKeys(select_query, query); /// Remove functions from ORDER BY if its argument is also in ORDER BY - if (settings.optimize_redundant_functions_in_order_by) + if (settings[Setting::optimize_redundant_functions_in_order_by]) optimizeRedundantFunctionsInOrderBy(select_query, context); /// Remove duplicate items from ORDER BY. @@ -688,7 +714,7 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, optimizeDuplicatesInOrderBy(select_query); /// If function "if" has String-type arguments, transform them into enum - if (settings.optimize_if_transform_strings_to_enum) + if (settings[Setting::optimize_if_transform_strings_to_enum]) transformIfStringsIntoEnum(query); /// Remove duplicated elements from LIMIT BY clause. @@ -697,10 +723,8 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, /// Remove duplicated columns from USING(...). optimizeUsing(select_query); - if (settings.optimize_or_like_chain - && settings.allow_hyperscan - && settings.max_hyperscan_regexp_length == 0 - && settings.max_hyperscan_regexp_total_length == 0) + if (settings[Setting::optimize_or_like_chain] && settings[Setting::allow_hyperscan] && settings[Setting::max_hyperscan_regexp_length] == 0 + && settings[Setting::max_hyperscan_regexp_total_length] == 0) { optimizeOrLikeChain(query); } diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index f31522ae649..ea08fd92339 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -65,6 +65,27 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool aggregate_functions_null_for_empty; + extern const SettingsBool any_join_distinct_right_table_keys; + extern const SettingsString count_distinct_implementation; + extern const SettingsBool enable_order_by_all; + extern const SettingsBool enable_positional_arguments; + extern const SettingsJoinStrictness join_default_strictness; + extern const SettingsBool legacy_column_name_of_tuple_literal; + extern const SettingsBool normalize_function_names; + extern const SettingsBool optimize_if_chain_to_multiif; + extern const SettingsBool optimize_group_by_function_keys; + extern const SettingsUInt64 optimize_min_equality_disjunction_chain_length; + extern const SettingsBool optimize_move_to_prewhere; + extern const SettingsBool optimize_multiif_to_if; + extern const SettingsBool optimize_normalize_count_variants; + extern const SettingsBool optimize_respect_aliases; + extern const SettingsBool optimize_trivial_count_query; + extern const SettingsBool rewrite_count_distinct_if_with_count_distinct_implementation; + extern const SettingsBool transform_null_in; +} namespace ErrorCodes { @@ -1327,7 +1348,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( renameDuplicatedColumns(select_query); /// Perform it before analyzing JOINs, because it may change number of columns with names unique and break some logic inside JOINs - if (settings.optimize_normalize_count_variants) + if (settings[Setting::optimize_normalize_count_variants]) TreeOptimizer::optimizeCountConstantAndSumOne(query, getContext()); if (tables_with_columns.size() > 1) @@ -1343,7 +1364,8 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns); /// Optimizes logical expressions. - LogicalExpressionsOptimizer(select_query, tables_with_columns, settings.optimize_min_equality_disjunction_chain_length.value).perform(); + LogicalExpressionsOptimizer(select_query, tables_with_columns, settings[Setting::optimize_min_equality_disjunction_chain_length].value) + .perform(); NameSet all_source_columns_set = source_columns_set; if (table_join) @@ -1359,7 +1381,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( expandGroupByAll(select_query); // expand ORDER BY ALL - if (settings.enable_order_by_all && select_query->order_by_all) + if (settings[Setting::enable_order_by_all] && select_query->order_by_all) expandOrderByAll(select_query, tables_with_columns); /// Remove unneeded columns according to 'required_result_columns'. @@ -1388,7 +1410,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( getContext()->getQueryContext()->addScalar(it.first, it.second); } - if (settings.legacy_column_name_of_tuple_literal) + if (settings[Setting::legacy_column_name_of_tuple_literal]) markTupleLiteralsAsLegacy(query); /// Push the predicate expression down to subqueries. The optimization should be applied to both initial and secondary queries. @@ -1399,8 +1421,8 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && !select_options.ignore_ast_optimizations; - bool optimize_multiif_to_if = ast_optimizations_allowed && settings.optimize_multiif_to_if; - TreeOptimizer::optimizeIf(query, result.aliases, settings.optimize_if_chain_to_multiif, optimize_multiif_to_if); + bool optimize_multiif_to_if_v = ast_optimizations_allowed && settings[Setting::optimize_multiif_to_if]; + TreeOptimizer::optimizeIf(query, result.aliases, settings[Setting::optimize_if_chain_to_multiif], optimize_multiif_to_if_v); if (ast_optimizations_allowed) TreeOptimizer::apply(query, result, tables_with_columns, getContext()); @@ -1408,8 +1430,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// array_join_alias_to_name, array_join_result_to_source. getArrayJoinedColumns(query, result, select_query, result.source_columns, source_columns_set); - setJoinStrictness( - *select_query, settings.join_default_strictness, settings.any_join_distinct_right_table_keys, result.analyzed_join); + setJoinStrictness(*select_query, settings[Setting::join_default_strictness], settings[Setting::any_join_distinct_right_table_keys], result.analyzed_join); auto * table_join_ast = select_query->join() ? select_query->join()->table_join->as() : nullptr; if (table_join_ast && tables_with_columns.size() >= 2) @@ -1432,7 +1453,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( /// rewrite filters for select query, must go after getArrayJoinedColumns bool is_initiator = getContext()->getClientInfo().distributed_depth == 0; - if (settings.optimize_respect_aliases && result.storage_snapshot && is_initiator) + if (settings[Setting::optimize_respect_aliases] && result.storage_snapshot && is_initiator) { std::unordered_set excluded_nodes; { @@ -1450,7 +1471,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( if (is_changed) { /// We should re-apply the optimization, because an expression substituted from alias column might be a function of a group key. - if (ast_optimizations_allowed && settings.optimize_group_by_function_keys) + if (ast_optimizations_allowed && settings[Setting::optimize_group_by_function_keys]) TreeOptimizer::optimizeGroupByFunctionKeys(select_query); result.aggregates = getAggregates(query, *select_query); @@ -1470,10 +1491,9 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( result.ast_join = select_query->join(); if (result.optimize_trivial_count) - result.optimize_trivial_count = settings.optimize_trivial_count_query && - !select_query->groupBy() && !select_query->having() && - !select_query->sampleSize() && !select_query->sampleOffset() && !select_query->final() && - (tables_with_columns.size() < 2 || isLeft(result.analyzed_join->kind())); + result.optimize_trivial_count = settings[Setting::optimize_trivial_count_query] && !select_query->groupBy() && !select_query->having() + && !select_query->sampleSize() && !select_query->sampleOffset() && !select_query->final() + && (tables_with_columns.size() < 2 || isLeft(result.analyzed_join->kind())); // remove outer braces in order by RewriteOrderByVisitor::Data data; @@ -1514,10 +1534,10 @@ TreeRewriterResultPtr TreeRewriter::analyze( getContext()->getQueryContext()->addScalar(it.first, it.second); } - if (settings.legacy_column_name_of_tuple_literal) + if (settings[Setting::legacy_column_name_of_tuple_literal]) markTupleLiteralsAsLegacy(query); - TreeOptimizer::optimizeIf(query, result.aliases, settings.optimize_if_chain_to_multiif, false); + TreeOptimizer::optimizeIf(query, result.aliases, settings[Setting::optimize_if_chain_to_multiif], false); if (allow_aggregations) { @@ -1553,31 +1573,31 @@ void TreeRewriter::normalize( if (!UserDefinedSQLFunctionFactory::instance().empty()) UserDefinedSQLFunctionVisitor::visit(query); - CustomizeCountDistinctVisitor::Data data_count_distinct{settings.count_distinct_implementation}; + CustomizeCountDistinctVisitor::Data data_count_distinct{settings[Setting::count_distinct_implementation]}; CustomizeCountDistinctVisitor(data_count_distinct).visit(query); - CustomizeCountIfDistinctVisitor::Data data_count_if_distinct{settings.count_distinct_implementation.toString() + "If"}; + CustomizeCountIfDistinctVisitor::Data data_count_if_distinct{settings[Setting::count_distinct_implementation].toString() + "If"}; CustomizeCountIfDistinctVisitor(data_count_if_distinct).visit(query); CustomizeIfDistinctVisitor::Data data_distinct_if{"DistinctIf"}; CustomizeIfDistinctVisitor(data_distinct_if).visit(query); - if (settings.rewrite_count_distinct_if_with_count_distinct_implementation) + if (settings[Setting::rewrite_count_distinct_if_with_count_distinct_implementation]) { - CustomizeCountDistinctIfVisitor::Data data_count_distinct_if{settings.count_distinct_implementation.toString() + "If"}; + CustomizeCountDistinctIfVisitor::Data data_count_distinct_if{settings[Setting::count_distinct_implementation].toString() + "If"}; CustomizeCountDistinctIfVisitor(data_count_distinct_if).visit(query); } ExistsExpressionVisitor::Data exists; ExistsExpressionVisitor(exists).visit(query); - if (context_->getSettingsRef().enable_positional_arguments) + if (context_->getSettingsRef()[Setting::enable_positional_arguments]) { ReplacePositionalArgumentsVisitor::Data data_replace_positional_arguments; ReplacePositionalArgumentsVisitor(data_replace_positional_arguments).visit(query); } - if (settings.transform_null_in) + if (settings[Setting::transform_null_in]) { CustomizeInVisitor::Data data_null_in{"nullIn"}; CustomizeInVisitor(data_null_in).visit(query); @@ -1593,7 +1613,7 @@ void TreeRewriter::normalize( } /// Rewrite all aggregate functions to add -OrNull suffix to them - if (settings.aggregate_functions_null_for_empty) + if (settings[Setting::aggregate_functions_null_for_empty]) { CustomizeAggregateFunctionsOrNullVisitor::Data data_or_null{"OrNull"}; CustomizeAggregateFunctionsOrNullVisitor(data_or_null).visit(query); @@ -1614,10 +1634,10 @@ void TreeRewriter::normalize( /// Notice: function name normalization is disabled when it's a secondary query, because queries are either /// already normalized on initiator node, or not normalized and should remain unnormalized for /// compatibility. - if (context_->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && settings.normalize_function_names) + if (context_->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && settings[Setting::normalize_function_names]) FunctionNameNormalizer::visit(query.get()); - if (settings.optimize_move_to_prewhere) + if (settings[Setting::optimize_move_to_prewhere]) { /// Required for PREWHERE ComparisonTupleEliminationVisitor::Data data_comparison_tuple_elimination; diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index d4bb0cc2f8a..4301ded40e1 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool normalize_function_names; +} namespace ErrorCodes { @@ -74,7 +79,8 @@ std::optional evaluateConstantExpressionImpl(c /// Notice: function name normalization is disabled when it's a secondary query, because queries are either /// already normalized on initiator node, or not normalized and should remain unnormalized for /// compatibility. - if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) + if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY + && context->getSettingsRef()[Setting::normalize_function_names]) FunctionNameNormalizer::visit(ast.get()); auto syntax_result = TreeRewriter(context, no_throw).analyze(ast, source_columns); diff --git a/src/Interpreters/examples/CMakeLists.txt b/src/Interpreters/examples/CMakeLists.txt index 4b1ec970b26..dff777af115 100644 --- a/src/Interpreters/examples/CMakeLists.txt +++ b/src/Interpreters/examples/CMakeLists.txt @@ -1,35 +1,35 @@ clickhouse_add_executable (hash_map hash_map.cpp) -target_link_libraries (hash_map PRIVATE dbms clickhouse_functions ch_contrib::sparsehash) +target_link_libraries (hash_map PRIVATE dbms clickhouse_functions clickhouse_functions ch_contrib::sparsehash) clickhouse_add_executable (hash_map_lookup hash_map_lookup.cpp) -target_link_libraries (hash_map_lookup PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression) +target_link_libraries (hash_map_lookup PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression) clickhouse_add_executable (hash_map3 hash_map3.cpp) -target_link_libraries (hash_map3 PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash) +target_link_libraries (hash_map3 PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash) clickhouse_add_executable (hash_map_string hash_map_string.cpp) -target_link_libraries (hash_map_string PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) +target_link_libraries (hash_map_string PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) clickhouse_add_executable (hash_map_string_2 hash_map_string_2.cpp) -target_link_libraries (hash_map_string_2 PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression) +target_link_libraries (hash_map_string_2 PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression) clickhouse_add_executable (hash_map_string_3 hash_map_string_3.cpp) -target_link_libraries (hash_map_string_3 PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash) +target_link_libraries (hash_map_string_3 PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash) clickhouse_add_executable (hash_map_string_small hash_map_string_small.cpp) -target_link_libraries (hash_map_string_small PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) +target_link_libraries (hash_map_string_small PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) clickhouse_add_executable (string_hash_map string_hash_map.cpp) -target_link_libraries (string_hash_map PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) +target_link_libraries (string_hash_map PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) clickhouse_add_executable (string_hash_map_aggregation string_hash_map.cpp) -target_link_libraries (string_hash_map_aggregation PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression) +target_link_libraries (string_hash_map_aggregation PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression) clickhouse_add_executable (string_hash_set string_hash_set.cpp) -target_link_libraries (string_hash_set PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression) +target_link_libraries (string_hash_set PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression) clickhouse_add_executable (two_level_hash_map two_level_hash_map.cpp) -target_link_libraries (two_level_hash_map PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) +target_link_libraries (two_level_hash_map PRIVATE dbms clickhouse_functions clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash) clickhouse_add_executable (jit_example jit_example.cpp) -target_link_libraries (jit_example PRIVATE dbms) +target_link_libraries (jit_example PRIVATE dbms clickhouse_functions) diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 1b57ad2b622..0eb703b34ec 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -29,6 +29,14 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_distributed_ddl; + extern const SettingsBool database_replicated_enforce_synchronous_settings; + extern const SettingsDistributedDDLOutputMode distributed_ddl_output_mode; + extern const SettingsInt64 distributed_ddl_task_timeout; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; +} namespace ErrorCodes { @@ -68,7 +76,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, { OpenTelemetry::SpanHolder span(__FUNCTION__, OpenTelemetry::SpanKind::PRODUCER); - if (context->getCurrentTransaction() && context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (context->getCurrentTransaction() && context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ON CLUSTER queries inside transactions are not supported"); /// Remove FORMAT and INTO OUTFILE if exists @@ -82,7 +90,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Distributed execution is not supported for such DDL queries"); } - if (!context->getSettingsRef().allow_distributed_ddl) + if (!context->getSettingsRef()[Setting::allow_distributed_ddl]) throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Distributed DDL queries are prohibited for the user"); if (const auto * query_alter = query_ptr->as()) @@ -249,14 +257,14 @@ private: BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & entry, ContextPtr context, const Strings * hosts_to_wait) { BlockIO io; - if (context->getSettingsRef().distributed_ddl_task_timeout == 0) + if (context->getSettingsRef()[Setting::distributed_ddl_task_timeout] == 0) return io; auto source = std::make_shared(node_path, entry, context, hosts_to_wait); io.pipeline = QueryPipeline(std::move(source)); - if (context->getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NONE || - context->getSettingsRef().distributed_ddl_output_mode == DistributedDDLOutputMode::NONE_ONLY_ACTIVE) + if (context->getSettingsRef()[Setting::distributed_ddl_output_mode] == DistributedDDLOutputMode::NONE + || context->getSettingsRef()[Setting::distributed_ddl_output_mode] == DistributedDDLOutputMode::NONE_ONLY_ACTIVE) io.pipeline.complete(std::make_shared(io.pipeline.getHeader())); return io; @@ -264,7 +272,7 @@ BlockIO getDistributedDDLStatus(const String & node_path, const DDLLogEntry & en Block DDLQueryStatusSource::getSampleBlock(ContextPtr context_, bool hosts_to_wait) { - auto output_mode = context_->getSettingsRef().distributed_ddl_output_mode; + auto output_mode = context_->getSettingsRef()[Setting::distributed_ddl_output_mode]; auto maybe_make_nullable = [&](const DataTypePtr & type) -> DataTypePtr { @@ -317,7 +325,7 @@ DDLQueryStatusSource::DDLQueryStatusSource( , watch(CLOCK_MONOTONIC_COARSE) , log(getLogger("DDLQueryStatusSource")) { - auto output_mode = context->getSettingsRef().distributed_ddl_output_mode; + auto output_mode = context->getSettingsRef()[Setting::distributed_ddl_output_mode]; throw_on_timeout = output_mode == DistributedDDLOutputMode::THROW || output_mode == DistributedDDLOutputMode::NONE; throw_on_timeout_only_active = output_mode == DistributedDDLOutputMode::THROW_ONLY_ACTIVE || output_mode == DistributedDDLOutputMode::NONE_ONLY_ACTIVE; @@ -336,7 +344,7 @@ DDLQueryStatusSource::DDLQueryStatusSource( } addTotalRowsApprox(waiting_hosts.size()); - timeout_seconds = context->getSettingsRef().distributed_ddl_task_timeout; + timeout_seconds = context->getSettingsRef()[Setting::distributed_ddl_task_timeout]; } std::pair DDLQueryStatusSource::parseHostAndPort(const String & host_id) const @@ -432,7 +440,7 @@ Chunk DDLQueryStatusSource::generate() return {}; String node_to_wait = "finished"; - if (is_replicated_database && context->getSettingsRef().database_replicated_enforce_synchronous_settings) + if (is_replicated_database && context->getSettingsRef()[Setting::database_replicated_enforce_synchronous_settings]) node_to_wait = "synced"; size_t try_number = 0; @@ -567,7 +575,7 @@ Chunk DDLQueryStatusSource::generate() if (status.code != 0 && !first_exception - && context->getSettingsRef().distributed_ddl_output_mode != DistributedDDLOutputMode::NEVER_THROW) + && context->getSettingsRef()[Setting::distributed_ddl_output_mode] != DistributedDDLOutputMode::NEVER_THROW) { if (is_replicated_database) throw Exception(ErrorCodes::LOGICAL_ERROR, "There was an error on {}: {} (probably it's a bug)", host_id, status.message); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 3260ea890c6..9fc3fbbddf9 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -95,6 +95,67 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsBool async_insert; + extern const SettingsBool calculate_text_stack_trace; + extern const SettingsBool deduplicate_blocks_in_dependent_materialized_views; + extern const SettingsDialect dialect; + extern const SettingsOverflowMode distinct_overflow_mode; + extern const SettingsBool enable_global_with_statement; + extern const SettingsBool enable_reads_from_query_cache; + extern const SettingsBool enable_writes_to_query_cache; + extern const SettingsSetOperationMode except_default_mode; + extern const SettingsOverflowModeGroupBy group_by_overflow_mode; + extern const SettingsBool implicit_transaction; + extern const SettingsSetOperationMode intersect_default_mode; + extern const SettingsOverflowMode join_overflow_mode; + extern const SettingsString log_comment; + extern const SettingsBool log_formatted_queries; + extern const SettingsBool log_processors_profiles; + extern const SettingsBool log_profile_events; + extern const SettingsUInt64 log_queries_cut_to_length; + extern const SettingsBool log_queries; + extern const SettingsMilliseconds log_queries_min_query_duration_ms; + extern const SettingsLogQueriesType log_queries_min_type; + extern const SettingsFloat log_queries_probability; + extern const SettingsBool log_query_settings; + extern const SettingsUInt64 max_ast_depth; + extern const SettingsUInt64 max_ast_elements; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsUInt64 max_result_bytes; + extern const SettingsUInt64 max_result_rows; + extern const SettingsUInt64 output_format_compression_level; + extern const SettingsUInt64 output_format_compression_zstd_window_log; + extern const SettingsBool query_cache_compress_entries; + extern const SettingsUInt64 query_cache_max_entries; + extern const SettingsUInt64 query_cache_max_size_in_bytes; + extern const SettingsMilliseconds query_cache_min_query_duration; + extern const SettingsUInt64 query_cache_min_query_runs; + extern const SettingsQueryCacheNondeterministicFunctionHandling query_cache_nondeterministic_function_handling; + extern const SettingsBool query_cache_share_between_users; + extern const SettingsBool query_cache_squash_partial_results; + extern const SettingsQueryCacheSystemTableHandling query_cache_system_table_handling; + extern const SettingsSeconds query_cache_ttl; + extern const SettingsOverflowMode read_overflow_mode; + extern const SettingsOverflowMode read_overflow_mode_leaf; + extern const SettingsOverflowMode result_overflow_mode; + extern const SettingsOverflowMode set_overflow_mode; + extern const SettingsOverflowMode sort_overflow_mode; + extern const SettingsBool throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; + extern const SettingsOverflowMode timeout_overflow_mode; + extern const SettingsOverflowMode transfer_overflow_mode; + extern const SettingsSetOperationMode union_default_mode; + extern const SettingsBool use_query_cache; + extern const SettingsBool wait_for_async_insert; + extern const SettingsSeconds wait_for_async_insert_timeout; +} namespace ErrorCodes { @@ -118,10 +179,10 @@ namespace FailPoints static void checkASTSizeLimits(const IAST & ast, const Settings & settings) { - if (settings.max_ast_depth) - ast.checkDepth(settings.max_ast_depth); - if (settings.max_ast_elements) - ast.checkSize(settings.max_ast_elements); + if (settings[Setting::max_ast_depth]) + ast.checkDepth(settings[Setting::max_ast_depth]); + if (settings[Setting::max_ast_elements]) + ast.checkSize(settings[Setting::max_ast_elements]); } @@ -140,8 +201,8 @@ static void logQuery(const String & query, ContextPtr context, bool internal, Qu const auto & initial_query_id = client_info.initial_query_id; const auto & current_user = client_info.current_user; - String comment = context->getSettingsRef().log_comment; - size_t max_query_size = context->getSettingsRef().max_query_size; + String comment = context->getSettingsRef()[Setting::log_comment]; + size_t max_query_size = context->getSettingsRef()[Setting::max_query_size]; if (comment.size() > max_query_size) comment.resize(max_query_size); @@ -328,7 +389,7 @@ QueryLogElement logQueryStart( elem.current_database = context->getCurrentDatabase(); elem.query = query_for_logging; - if (settings.log_formatted_queries) + if (settings[Setting::log_formatted_queries]) elem.formatted_query = queryToString(query_ast); elem.normalized_query_hash = normalizedQueryHash(query_for_logging, false); elem.query_kind = query_ast->getQueryKind(); @@ -338,7 +399,7 @@ QueryLogElement logQueryStart( if (auto txn = context->getCurrentTransaction()) elem.tid = txn->tid; - bool log_queries = settings.log_queries && !internal; + bool log_queries = settings[Setting::log_queries] && !internal; /// Log into system table start of query execution, if need. if (log_queries) @@ -361,14 +422,14 @@ QueryLogElement logQueryStart( else if (interpreter) interpreter->extendQueryLogElem(elem, query_ast, context, query_database, query_table); - if (settings.log_query_settings) + if (settings[Setting::log_query_settings]) elem.query_settings = std::make_shared(context->getSettingsRef()); - elem.log_comment = settings.log_comment; - if (elem.log_comment.size() > settings.max_query_size) - elem.log_comment.resize(settings.max_query_size); + elem.log_comment = settings[Setting::log_comment]; + if (elem.log_comment.size() > settings[Setting::max_query_size]) + elem.log_comment.resize(settings[Setting::max_query_size]); - if (elem.type >= settings.log_queries_min_type && !settings.log_queries_min_query_duration_ms.totalMilliseconds()) + if (elem.type >= settings[Setting::log_queries_min_type] && !settings[Setting::log_queries_min_query_duration_ms].totalMilliseconds()) { if (auto query_log = context->getQueryLog()) query_log->add(elem); @@ -389,10 +450,7 @@ void logQueryFinish( bool internal) { const Settings & settings = context->getSettingsRef(); - auto log_queries = settings.log_queries && !internal; - auto log_queries_min_type = settings.log_queries_min_type; - auto log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(); - auto log_processors_profiles = settings.log_processors_profiles; + auto log_queries = settings[Setting::log_queries] && !internal; QueryStatusPtr process_list_elem = context->getProcessListElement(); if (process_list_elem) @@ -402,7 +460,7 @@ void logQueryFinish( std::shared_ptr profile_counters; QueryStatusInfo info = process_list_elem->getInfo(true, true); - if (context->getSettingsRef().log_profile_events) + if (settings[Setting::log_profile_events]) profile_counters = info.profile_counters; else profile_counters.swap(info.profile_counters); @@ -446,13 +504,13 @@ void logQueryFinish( elem.query_cache_usage = query_cache_usage; - if (log_queries && elem.type >= log_queries_min_type - && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) + if (log_queries && elem.type >= settings[Setting::log_queries_min_type] + && static_cast(elem.query_duration_ms) >= settings[Setting::log_queries_min_query_duration_ms].totalMilliseconds()) { if (auto query_log = context->getQueryLog()) query_log->add(elem); } - if (log_processors_profiles) + if (settings[Setting::log_processors_profiles]) { if (auto processors_profile_log = context->getProcessorsProfileLog()) { @@ -526,9 +584,7 @@ void logQueryException( bool log_error) { const Settings & settings = context->getSettingsRef(); - auto log_queries = settings.log_queries && !internal; - auto log_queries_min_type = settings.log_queries_min_type; - auto log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(); + auto log_queries = settings[Setting::log_queries] && !internal; elem.type = QueryLogElementType::EXCEPTION_WHILE_PROCESSING; elem.exception_code = getCurrentExceptionCode(); @@ -547,7 +603,7 @@ void logQueryException( if (process_list_elem) { - QueryStatusInfo info = process_list_elem->getInfo(true, settings.log_profile_events, false); + QueryStatusInfo info = process_list_elem->getInfo(true, settings[Setting::log_profile_events], false); addStatusInfoToQueryLogElement(elem, info, query_ast, context); } else @@ -557,12 +613,13 @@ void logQueryException( elem.query_cache_usage = QueryCache::Usage::None; - if (settings.calculate_text_stack_trace && log_error) + if (settings[Setting::calculate_text_stack_trace] && log_error) setExceptionStackTrace(elem); logException(context, elem, log_error); /// In case of exception we log internal queries also - if (log_queries && elem.type >= log_queries_min_type && static_cast(elem.query_duration_ms) >= log_queries_min_query_duration_ms) + if (log_queries && elem.type >= settings[Setting::log_queries_min_type] + && static_cast(elem.query_duration_ms) >= settings[Setting::log_queries_min_query_duration_ms].totalMilliseconds()) { if (auto query_log = context->getQueryLog()) query_log->add(elem); @@ -619,7 +676,7 @@ void logExceptionBeforeStart( if (ast) { elem.query_kind = ast->getQueryKind(); - if (settings.log_formatted_queries) + if (settings[Setting::log_formatted_queries]) elem.formatted_query = queryToString(ast); } @@ -635,24 +692,25 @@ void logExceptionBeforeStart( elem.client_info = context->getClientInfo(); - elem.log_comment = settings.log_comment; - if (elem.log_comment.size() > settings.max_query_size) - elem.log_comment.resize(settings.max_query_size); + elem.log_comment = settings[Setting::log_comment]; + if (elem.log_comment.size() > settings[Setting::max_query_size]) + elem.log_comment.resize(settings[Setting::max_query_size]); if (auto txn = context->getCurrentTransaction()) elem.tid = txn->tid; - if (settings.log_query_settings) - elem.query_settings = std::make_shared(context->getSettingsRef()); + if (settings[Setting::log_query_settings]) + elem.query_settings = std::make_shared(settings); - if (settings.calculate_text_stack_trace) + if (settings[Setting::calculate_text_stack_trace]) setExceptionStackTrace(elem); logException(context, elem); /// Update performance counters before logging to query_log CurrentThread::finalizePerformanceCounters(); - if (settings.log_queries && elem.type >= settings.log_queries_min_type && !settings.log_queries_min_query_duration_ms.totalMilliseconds()) + if (settings[Setting::log_queries] && elem.type >= settings[Setting::log_queries_min_type] + && !settings[Setting::log_queries_min_query_duration_ms].totalMilliseconds()) if (auto query_log = context->getQueryLog()) query_log->add(elem); @@ -762,7 +820,7 @@ static std::tuple executeQueryImpl( const Settings & settings = context->getSettingsRef(); - size_t max_query_size = settings.max_query_size; + size_t max_query_size = settings[Setting::max_query_size]; /// Don't limit the size of internal queries or distributed subquery. if (internal || client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) max_query_size = 0; @@ -770,27 +828,27 @@ static std::tuple executeQueryImpl( ASTPtr ast; String query; String query_for_logging; - size_t log_queries_cut_to_length = context->getSettingsRef().log_queries_cut_to_length; + size_t log_queries_cut_to_length = settings[Setting::log_queries_cut_to_length]; /// Parse the query from string. try { - if (settings.dialect == Dialect::kusto && !internal) + if (settings[Setting::dialect] == Dialect::kusto && !internal) { - ParserKQLStatement parser(end, settings.allow_settings_after_format_in_insert); + ParserKQLStatement parser(end, settings[Setting::allow_settings_after_format_in_insert]); /// TODO: parser should fail early when max_query_size limit is reached. - ast = parseKQLQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ast = parseKQLQuery(parser, begin, end, "", max_query_size, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } - else if (settings.dialect == Dialect::prql && !internal) + else if (settings[Setting::dialect] == Dialect::prql && !internal) { - ParserPRQLQuery parser(max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); - ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ParserPRQLQuery parser(max_query_size, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); + ast = parseQuery(parser, begin, end, "", max_query_size, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } else { - ParserQuery parser(end, settings.allow_settings_after_format_in_insert); + ParserQuery parser(end, settings[Setting::allow_settings_after_format_in_insert]); /// TODO: parser should fail early when max_query_size limit is reached. - ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ast = parseQuery(parser, begin, end, "", max_query_size, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); #ifndef NDEBUG /// Verify that AST formatting is consistent: @@ -804,10 +862,14 @@ static std::tuple executeQueryImpl( ASTPtr ast2; try { - ast2 = parseQuery(parser, + ast2 = parseQuery( + parser, formatted1.data(), formatted1.data() + formatted1.size(), - "", new_max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + "", + new_max_query_size, + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); } catch (const Exception & e) { @@ -928,7 +990,7 @@ static std::tuple executeQueryImpl( /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter), /// to allow settings to take effect. InterpreterSetQuery::applySettingsFromQuery(ast, context); - validateAnalyzerSettings(ast, context->getSettingsRef().allow_experimental_analyzer); + validateAnalyzerSettings(ast, settings[Setting::allow_experimental_analyzer]); if (auto * insert_query = ast->as()) insert_query->tail = istr; @@ -937,9 +999,9 @@ static std::tuple executeQueryImpl( /// If it is used - do the random sampling and "collapse" the settings. /// It allows to consistently log queries with all the subqueries in distributed query processing /// (subqueries on remote nodes will receive these "collapsed" settings) - if (!internal && settings.log_queries && settings.log_queries_probability < 1.0) + if (!internal && settings[Setting::log_queries] && settings[Setting::log_queries_probability] < 1.0) { - std::bernoulli_distribution should_write_log{settings.log_queries_probability}; + std::bernoulli_distribution should_write_log{settings[Setting::log_queries_probability]}; context->setSetting("log_queries", should_write_log(thread_local_rng)); context->setSetting("log_queries_probability", 1.0); @@ -954,19 +1016,19 @@ static std::tuple executeQueryImpl( logQuery(query_for_logging, context, internal, stage); /// Propagate WITH statement to children ASTSelect. - if (settings.enable_global_with_statement) + if (settings[Setting::enable_global_with_statement]) { ApplyWithGlobalVisitor::visit(ast); } { - SelectIntersectExceptQueryVisitor::Data data{settings.intersect_default_mode, settings.except_default_mode}; + SelectIntersectExceptQueryVisitor::Data data{settings[Setting::intersect_default_mode], settings[Setting::except_default_mode]}; SelectIntersectExceptQueryVisitor{data}.visit(ast); } { /// Normalize SelectWithUnionQuery - NormalizeSelectWithUnionQueryVisitor::Data data{settings.union_default_mode}; + NormalizeSelectWithUnionQueryVisitor::Data data{settings[Setting::union_default_mode]}; NormalizeSelectWithUnionQueryVisitor{data}.visit(ast); } @@ -985,7 +1047,7 @@ static std::tuple executeQueryImpl( context->initializeExternalTablesIfSet(); auto * insert_query = ast->as(); - bool async_insert_enabled = settings.async_insert; + bool async_insert_enabled = settings[Setting::async_insert]; /// Resolve database before trying to use async insert feature - to properly hash the query. if (insert_query) @@ -1052,9 +1114,9 @@ static std::tuple executeQueryImpl( if (async_insert) { - if (context->getCurrentTransaction() && settings.throw_on_unsupported_query_inside_transaction) + if (context->getCurrentTransaction() && settings[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Async inserts inside transactions are not supported"); - if (settings.implicit_transaction && settings.throw_on_unsupported_query_inside_transaction) + if (settings[Setting::implicit_transaction] && settings[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Async inserts with 'implicit_transaction' are not supported"); /// Let's agree on terminology and say that a mini-INSERT is an asynchronous INSERT @@ -1066,8 +1128,8 @@ static std::tuple executeQueryImpl( /// The process of forming such blocks is not deterministic so each time we retry mini-INSERTs the resulting /// block may be concatenated differently. /// That's why deduplication in dependent Materialized Views doesn't make sense in presence of async INSERTs. - if (settings.throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert && - settings.deduplicate_blocks_in_dependent_materialized_views) + if (settings[Setting::throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert] + && settings[Setting::deduplicate_blocks_in_dependent_materialized_views]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Deduplication in dependent materialized view cannot work together with async inserts. "\ "Please disable either `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting."); @@ -1085,9 +1147,9 @@ static std::tuple executeQueryImpl( if (result.status == AsynchronousInsertQueue::PushResult::OK) { - if (settings.wait_for_async_insert) + if (settings[Setting::wait_for_async_insert]) { - auto timeout = settings.wait_for_async_insert_timeout.totalMilliseconds(); + auto timeout = settings[Setting::wait_for_async_insert_timeout].totalMilliseconds(); auto source = std::make_shared(std::move(result.future), timeout); res.pipeline = QueryPipeline(Pipe(std::move(source))); } @@ -1115,9 +1177,7 @@ static std::tuple executeQueryImpl( } QueryCachePtr query_cache = context->getQueryCache(); - const bool can_use_query_cache = query_cache != nullptr - && settings.use_query_cache - && !internal + const bool can_use_query_cache = query_cache != nullptr && settings[Setting::use_query_cache] && !internal && client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY && (ast->as() || ast->as()); QueryCache::Usage query_cache_usage = QueryCache::Usage::None; @@ -1125,16 +1185,16 @@ static std::tuple executeQueryImpl( /// Bug 67476: If the query runs with a non-THROW overflow mode and hits a limit, the query cache will store a truncated result (if /// enabled). This is incorrect. Unfortunately it is hard to detect from the perspective of the query cache that the query result /// is truncated. Therefore throw an exception, to notify the user to disable either the query cache or use another overflow mode. - if (settings.use_query_cache && (settings.read_overflow_mode != OverflowMode::THROW - || settings.read_overflow_mode_leaf != OverflowMode::THROW - || settings.group_by_overflow_mode != OverflowMode::THROW - || settings.sort_overflow_mode != OverflowMode::THROW - || settings.result_overflow_mode != OverflowMode::THROW - || settings.timeout_overflow_mode != OverflowMode::THROW - || settings.set_overflow_mode != OverflowMode::THROW - || settings.join_overflow_mode != OverflowMode::THROW - || settings.transfer_overflow_mode != OverflowMode::THROW - || settings.distinct_overflow_mode != OverflowMode::THROW)) + if (settings[Setting::use_query_cache] && (settings[Setting::read_overflow_mode] != OverflowMode::THROW + || settings[Setting::read_overflow_mode_leaf] != OverflowMode::THROW + || settings[Setting::group_by_overflow_mode] != OverflowMode::THROW + || settings[Setting::sort_overflow_mode] != OverflowMode::THROW + || settings[Setting::result_overflow_mode] != OverflowMode::THROW + || settings[Setting::timeout_overflow_mode] != OverflowMode::THROW + || settings[Setting::set_overflow_mode] != OverflowMode::THROW + || settings[Setting::join_overflow_mode] != OverflowMode::THROW + || settings[Setting::transfer_overflow_mode] != OverflowMode::THROW + || settings[Setting::distinct_overflow_mode] != OverflowMode::THROW)) throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_NON_THROW_OVERFLOW_MODE, "use_query_cache and overflow_mode != 'throw' cannot be used together"); /// If the query runs with "use_query_cache = 1", we first probe if the query cache already contains the query result (if yes: @@ -1152,7 +1212,7 @@ static std::tuple executeQueryImpl( /// a pipeline with a source populated by the query cache. auto get_result_from_query_cache = [&]() { - if (can_use_query_cache && settings.enable_reads_from_query_cache) + if (can_use_query_cache && settings[Setting::enable_reads_from_query_cache]) { QueryCache::Key key(ast, context->getCurrentDatabase(), *settings_copy, context->getUserID(), context->getCurrentRoles()); QueryCache::Reader reader = query_cache->createReader(key); @@ -1171,7 +1231,7 @@ static std::tuple executeQueryImpl( if (!get_result_from_query_cache()) { /// We need to start the (implicit) transaction before getting the interpreter as this will get links to the latest snapshots - if (!context->getCurrentTransaction() && settings.implicit_transaction && !ast->as()) + if (!context->getCurrentTransaction() && settings[Setting::implicit_transaction] && !ast->as()) { try { @@ -1190,7 +1250,7 @@ static std::tuple executeQueryImpl( interpreter = InterpreterFactory::instance().get(ast, context, SelectQueryOptions(stage).setInternal(internal)); const auto & query_settings = context->getSettingsRef(); - if (context->getCurrentTransaction() && query_settings.throw_on_unsupported_query_inside_transaction) + if (context->getCurrentTransaction() && query_settings[Setting::throw_on_unsupported_query_inside_transaction]) { if (!interpreter->supportsTransactions()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Transactions are not supported for this type of query ({})", ast->getID()); @@ -1223,7 +1283,7 @@ static std::tuple executeQueryImpl( if (!interpreter->ignoreLimits()) { limits.mode = LimitsMode::LIMITS_CURRENT; - limits.size_limits = SizeLimits(settings.max_result_rows, settings.max_result_bytes, settings.result_overflow_mode); + limits.size_limits = SizeLimits(settings[Setting::max_result_rows], settings[Setting::max_result_bytes], settings[Setting::result_overflow_mode]); } if (auto * insert_interpreter = typeid_cast(&*interpreter)) @@ -1255,15 +1315,16 @@ static std::tuple executeQueryImpl( /// If it is a non-internal SELECT query, and active (write) use of the query cache is enabled, then add a processor on /// top of the pipeline which stores the result in the query cache. - if (can_use_query_cache && settings.enable_writes_to_query_cache) + if (can_use_query_cache && settings[Setting::enable_writes_to_query_cache]) { /// Only use the query cache if the query does not contain non-deterministic functions or system tables (which are typically non-deterministic) const bool ast_contains_nondeterministic_functions = astContainsNonDeterministicFunctions(ast, context); const bool ast_contains_system_tables = astContainsSystemTables(ast, context); - const QueryCacheNondeterministicFunctionHandling nondeterministic_function_handling = settings.query_cache_nondeterministic_function_handling; - const QueryCacheSystemTableHandling system_table_handling = settings.query_cache_system_table_handling; + const QueryCacheNondeterministicFunctionHandling nondeterministic_function_handling + = settings[Setting::query_cache_nondeterministic_function_handling]; + const QueryCacheSystemTableHandling system_table_handling = settings[Setting::query_cache_system_table_handling]; if (ast_contains_nondeterministic_functions && nondeterministic_function_handling == QueryCacheNondeterministicFunctionHandling::Throw) throw Exception(ErrorCodes::QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS, @@ -1281,26 +1342,26 @@ static std::tuple executeQueryImpl( QueryCache::Key key( ast, context->getCurrentDatabase(), *settings_copy, res.pipeline.getHeader(), context->getUserID(), context->getCurrentRoles(), - settings.query_cache_share_between_users, - std::chrono::system_clock::now() + std::chrono::seconds(settings.query_cache_ttl), - settings.query_cache_compress_entries); + settings[Setting::query_cache_share_between_users], + std::chrono::system_clock::now() + std::chrono::seconds(settings[Setting::query_cache_ttl]), + settings[Setting::query_cache_compress_entries]); - const size_t num_query_runs = settings.query_cache_min_query_runs ? query_cache->recordQueryRun(key) : 1; /// try to avoid locking a mutex in recordQueryRun() - if (num_query_runs <= settings.query_cache_min_query_runs) + const size_t num_query_runs = settings[Setting::query_cache_min_query_runs] ? query_cache->recordQueryRun(key) : 1; /// try to avoid locking a mutex in recordQueryRun() + if (num_query_runs <= settings[Setting::query_cache_min_query_runs]) { LOG_TRACE(getLogger("QueryCache"), "Skipped insert because the query ran {} times but the minimum required number of query runs to cache the query result is {}", - num_query_runs, settings.query_cache_min_query_runs); + num_query_runs, settings[Setting::query_cache_min_query_runs]); } else { auto query_cache_writer = std::make_shared(query_cache->createWriter( key, - std::chrono::milliseconds(settings.query_cache_min_query_duration.totalMilliseconds()), - settings.query_cache_squash_partial_results, - settings.max_block_size, - settings.query_cache_max_size_in_bytes, - settings.query_cache_max_entries)); + std::chrono::milliseconds(settings[Setting::query_cache_min_query_duration].totalMilliseconds()), + settings[Setting::query_cache_squash_partial_results], + settings[Setting::max_block_size], + settings[Setting::query_cache_max_size_in_bytes], + settings[Setting::query_cache_max_entries])); res.pipeline.writeResultIntoQueryCache(query_cache_writer); query_cache_usage = QueryCache::Usage::Write; } @@ -1457,7 +1518,7 @@ void executeQuery( throw; } - size_t max_query_size = context->getSettingsRef().max_query_size; + size_t max_query_size = context->getSettingsRef()[Setting::max_query_size]; if (istr.buffer().end() - istr.position() > static_cast(max_query_size)) { @@ -1601,9 +1662,8 @@ void executeQuery( compressed_buffer = wrapWriteBufferWithCompressionMethod( std::make_unique(out_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_EXCL | O_CREAT), chooseCompressionMethod(out_file, compression_method), - /* compression level = */ static_cast(settings.output_format_compression_level), - /* zstd_window_log = */ static_cast(settings.output_format_compression_zstd_window_log) - ); + /* compression level = */ static_cast(settings[Setting::output_format_compression_level]), + /* zstd_window_log = */ static_cast(settings[Setting::output_format_compression_zstd_window_log])); } format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.cpp b/src/Interpreters/formatWithPossiblyHidingSecrets.cpp new file mode 100644 index 00000000000..ba02a9f164b --- /dev/null +++ b/src/Interpreters/formatWithPossiblyHidingSecrets.cpp @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +namespace DB +{ +namespace Setting +{ + extern const SettingsBool format_display_secrets_in_show_and_select; + extern const SettingsBool output_format_always_quote_identifiers; + extern const SettingsIdentifierQuotingStyle output_format_identifier_quoting_style; + extern const SettingsBool print_pretty_type_names; +} + +String format(const SecretHidingFormatSettings & settings) +{ + const bool show_secrets = settings.ctx->displaySecretsInShowAndSelect() + && settings.ctx->getSettingsRef()[Setting::format_display_secrets_in_show_and_select] + && settings.ctx->getAccess()->isGranted(AccessType::displaySecretsInShowAndSelect); + + return settings.query.formatWithPossiblyHidingSensitiveData( + settings.max_length, + settings.one_line, + show_secrets, + settings.ctx->getSettingsRef()[Setting::print_pretty_type_names], + settings.ctx->getSettingsRef()[Setting::output_format_always_quote_identifiers], + settings.ctx->getSettingsRef()[Setting::output_format_identifier_quoting_style]); +} + +} diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.h b/src/Interpreters/formatWithPossiblyHidingSecrets.h index 61f689fb821..db88fdb2b92 100644 --- a/src/Interpreters/formatWithPossiblyHidingSecrets.h +++ b/src/Interpreters/formatWithPossiblyHidingSecrets.h @@ -1,13 +1,10 @@ #pragma once -#include -#include - - -#include +#include namespace DB { +class IAST; struct SecretHidingFormatSettings { @@ -19,19 +16,5 @@ struct SecretHidingFormatSettings bool one_line = true; }; -inline String format(const SecretHidingFormatSettings & settings) -{ - const bool show_secrets = settings.ctx->displaySecretsInShowAndSelect() - && settings.ctx->getSettingsRef().format_display_secrets_in_show_and_select - && settings.ctx->getAccess()->isGranted(AccessType::displaySecretsInShowAndSelect); - - return settings.query.formatWithPossiblyHidingSensitiveData( - settings.max_length, - settings.one_line, - show_secrets, - settings.ctx->getSettingsRef().print_pretty_type_names, - settings.ctx->getSettingsRef().output_format_always_quote_identifiers, - settings.ctx->getSettingsRef().output_format_identifier_quoting_style); -} - +String format(const SecretHidingFormatSettings & settings); } diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp index 6f902c7cd7e..5f0e9b9671f 100644 --- a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp @@ -15,6 +15,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} namespace ErrorCodes { @@ -168,8 +174,13 @@ ASTPtr parseCustomKeyForTable(const String & custom_key, const Context & context ParserExpression parser; const auto & settings = context.getSettingsRef(); return parseQuery( - parser, custom_key.data(), custom_key.data() + custom_key.size(), - "parallel replicas custom key", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parser, + custom_key.data(), + custom_key.data() + custom_key.size(), + "parallel replicas custom key", + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); } } diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp index cf18cbbb54a..c0b691591e8 100644 --- a/src/Interpreters/getHeaderForProcessingStage.cpp +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -16,6 +16,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} namespace ErrorCodes { @@ -141,7 +145,7 @@ Block getHeaderForProcessingStage( Block result; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { auto storage = std::make_shared(storage_snapshot->storage.getStorageID(), storage_snapshot->getAllColumnsDescription(), diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp index 909875b99a0..a6c3c090247 100644 --- a/src/Interpreters/interpretSubquery.cpp +++ b/src/Interpreters/interpretSubquery.cpp @@ -18,6 +18,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool extremes; + extern const SettingsUInt64 max_result_bytes; + extern const SettingsUInt64 max_result_rows; +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -63,10 +70,10 @@ std::shared_ptr interpretSubquery( */ auto subquery_context = Context::createCopy(context); Settings subquery_settings = context->getSettingsCopy(); - subquery_settings.max_result_rows = 0; - subquery_settings.max_result_bytes = 0; + subquery_settings[Setting::max_result_rows] = 0; + subquery_settings[Setting::max_result_bytes] = 0; /// The calculation of `extremes` does not make sense and is not necessary (if you do it, then the `extremes` of the subquery can be taken instead of the whole query). - subquery_settings.extremes = false; + subquery_settings[Setting::extremes] = false; subquery_context->setSettings(subquery_settings); auto subquery_options = options.subquery(); diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index 4d17891f9f8..8eaf26672e2 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -34,6 +34,12 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_database_ordinary; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -60,8 +66,13 @@ static void executeCreateQuery( const Settings & settings = context->getSettingsRef(); ParserCreateQuery parser; ASTPtr ast = parseQuery( - parser, query.data(), query.data() + query.size(), "in file " + file_name, - 0, settings.max_parser_depth, settings.max_parser_backtracks); + parser, + query.data(), + query.data() + query.size(), + "in file " + file_name, + 0, + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); auto & ast_create_query = ast->as(); ast_create_query.setDatabase(database); @@ -474,7 +485,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons void maybeConvertSystemDatabase(ContextMutablePtr context, LoadTaskPtrs & system_startup_tasks) { /// TODO remove this check, convert system database unconditionally - if (context->getSettingsRef().allow_deprecated_database_ordinary) + if (context->getSettingsRef()[Setting::allow_deprecated_database_ordinary]) return; maybeConvertOrdinaryDatabaseToAtomic(context, DatabaseCatalog::SYSTEM_DATABASE, &system_startup_tasks); diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 0c6d18dca70..13d10c7d7e7 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -14,6 +14,20 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_dynamic_type; + extern const SettingsBool allow_experimental_json_type; + extern const SettingsBool allow_experimental_object_type; + extern const SettingsBool allow_experimental_variant_type; + extern const SettingsBool allow_suspicious_fixed_string_types; + extern const SettingsBool allow_suspicious_low_cardinality_types; + extern const SettingsBool allow_suspicious_variant_types; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsBool validate_experimental_and_suspicious_types_inside_nested_types; +} namespace ErrorCodes { @@ -23,15 +37,15 @@ extern const int ILLEGAL_COLUMN; } -DataTypeValidationSettings::DataTypeValidationSettings(const DB::Settings& settings) - : allow_suspicious_low_cardinality_types(settings.allow_suspicious_low_cardinality_types) - , allow_experimental_object_type(settings.allow_experimental_object_type) - , allow_suspicious_fixed_string_types(settings.allow_suspicious_fixed_string_types) - , allow_experimental_variant_type(settings.allow_experimental_variant_type) - , allow_suspicious_variant_types(settings.allow_suspicious_variant_types) - , validate_nested_types(settings.validate_experimental_and_suspicious_types_inside_nested_types) - , allow_experimental_dynamic_type(settings.allow_experimental_dynamic_type) - , allow_experimental_json_type(settings.allow_experimental_json_type) +DataTypeValidationSettings::DataTypeValidationSettings(const DB::Settings & settings) + : allow_suspicious_low_cardinality_types(settings[Setting::allow_suspicious_low_cardinality_types]) + , allow_experimental_object_type(settings[Setting::allow_experimental_object_type]) + , allow_suspicious_fixed_string_types(settings[Setting::allow_suspicious_fixed_string_types]) + , allow_experimental_variant_type(settings[Setting::allow_experimental_variant_type]) + , allow_suspicious_variant_types(settings[Setting::allow_suspicious_variant_types]) + , validate_nested_types(settings[Setting::validate_experimental_and_suspicious_types_inside_nested_types]) + , allow_experimental_dynamic_type(settings[Setting::allow_experimental_dynamic_type]) + , allow_experimental_json_type(settings[Setting::allow_experimental_json_type]) { } @@ -159,7 +173,13 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con ParserColumnDeclarationList parser(true, true); const Settings & settings = context->getSettingsRef(); - ASTPtr columns_list_raw = parseQuery(parser, structure, "columns declaration list", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ASTPtr columns_list_raw = parseQuery( + parser, + structure, + "columns declaration list", + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); auto * columns_list = dynamic_cast(columns_list_raw.get()); if (!columns_list) @@ -180,7 +200,17 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip const char * start = structure.data(); const char * end = structure.data() + structure.size(); ASTPtr columns_list_raw = tryParseQuery( - parser, start, end, error, false, "columns declaration list", false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks, true); + parser, + start, + end, + error, + false, + "columns declaration list", + false, + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks], + true); if (!columns_list_raw) return false; diff --git a/src/Interpreters/removeOnClusterClauseIfNeeded.cpp b/src/Interpreters/removeOnClusterClauseIfNeeded.cpp index a6595d330fc..6d93fad318f 100644 --- a/src/Interpreters/removeOnClusterClauseIfNeeded.cpp +++ b/src/Interpreters/removeOnClusterClauseIfNeeded.cpp @@ -24,7 +24,12 @@ namespace DB { - +namespace Setting +{ + extern const SettingsBool ignore_on_cluster_for_replicated_named_collections_queries; + extern const SettingsBool ignore_on_cluster_for_replicated_access_entities_queries; + extern const SettingsBool ignore_on_cluster_for_replicated_udf_queries; +} static bool isUserDefinedFunctionQuery(const ASTPtr & query) { @@ -58,13 +63,13 @@ ASTPtr removeOnClusterClauseIfNeeded(const ASTPtr & query, ContextPtr context, c return query; if ((isUserDefinedFunctionQuery(query) - && context->getSettingsRef().ignore_on_cluster_for_replicated_udf_queries + && context->getSettingsRef()[Setting::ignore_on_cluster_for_replicated_udf_queries] && context->getUserDefinedSQLObjectsStorage().isReplicated()) || (isAccessControlQuery(query) - && context->getSettingsRef().ignore_on_cluster_for_replicated_access_entities_queries + && context->getSettingsRef()[Setting::ignore_on_cluster_for_replicated_access_entities_queries] && context->getAccessControl().containsStorage(ReplicatedAccessStorage::STORAGE_TYPE)) || (isNamedCollectionQuery(query) - && context->getSettingsRef().ignore_on_cluster_for_replicated_named_collections_queries + && context->getSettingsRef()[Setting::ignore_on_cluster_for_replicated_named_collections_queries] && NamedCollectionFactory::instance().usesReplicatedStorage())) { LOG_DEBUG(getLogger("removeOnClusterClauseIfNeeded"), "ON CLUSTER clause was ignored for query {}", query->getID()); diff --git a/src/Parsers/examples/CMakeLists.txt b/src/Parsers/examples/CMakeLists.txt index 07f60601acd..0ea52497ecc 100644 --- a/src/Parsers/examples/CMakeLists.txt +++ b/src/Parsers/examples/CMakeLists.txt @@ -4,7 +4,7 @@ clickhouse_add_executable(lexer lexer.cpp ${SRCS}) target_link_libraries(lexer PRIVATE clickhouse_parsers clickhouse_common_config) clickhouse_add_executable(select_parser select_parser.cpp ${SRCS} "../../Server/ServerType.cpp") -target_link_libraries(select_parser PRIVATE dbms) +target_link_libraries(select_parser PRIVATE dbms clickhouse_functions) clickhouse_add_executable(create_parser create_parser.cpp ${SRCS} "../../Server/ServerType.cpp") -target_link_libraries(create_parser PRIVATE dbms) +target_link_libraries(create_parser PRIVATE dbms clickhouse_functions) diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index 9cac4e61380..57d29ab0768 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -19,6 +19,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool transform_null_in; +} namespace ErrorCodes { @@ -70,16 +74,13 @@ public: else if (const auto * constant_node = in_second_argument->as()) { auto set = getSetElementsForConstantValue( - in_first_argument->getResultType(), - constant_node->getValue(), - constant_node->getResultType(), - settings.transform_null_in); + in_first_argument->getResultType(), constant_node->getValue(), constant_node->getResultType(), settings[Setting::transform_null_in]); DataTypes set_element_types = {in_first_argument->getResultType()}; const auto * left_tuple_type = typeid_cast(set_element_types.front().get()); if (left_tuple_type && left_tuple_type->getElements().size() != 1) set_element_types = left_tuple_type->getElements(); - set_element_types = Set::getElementTypes(std::move(set_element_types), settings.transform_null_in); + set_element_types = Set::getElementTypes(std::move(set_element_types), settings[Setting::transform_null_in]); auto set_key = in_second_argument->getTreeHash(); if (sets.findTuple(set_key, set_element_types)) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 7ed16f17087..688c6045a3a 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -92,6 +92,50 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 aggregation_in_order_max_block_bytes; + extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsBool collect_hash_table_stats_during_aggregation; + extern const SettingsBool compile_aggregate_expressions; + extern const SettingsOverflowMode distinct_overflow_mode; + extern const SettingsBool distributed_aggregation_memory_efficient; + extern const SettingsBool enable_memory_bound_merging_of_aggregation_results; + extern const SettingsBool enable_software_prefetch_in_aggregation; + extern const SettingsBool empty_result_for_aggregation_by_constant_keys_on_empty_set; + extern const SettingsBool empty_result_for_aggregation_by_empty_set; + extern const SettingsBool exact_rows_before_limit; + extern const SettingsBool extremes; + extern const SettingsBool force_aggregation_in_order; + extern const SettingsOverflowModeGroupBy group_by_overflow_mode; + extern const SettingsUInt64 group_by_two_level_threshold; + extern const SettingsUInt64 group_by_two_level_threshold_bytes; + extern const SettingsBool group_by_use_nulls; + extern const SettingsUInt64 max_bytes_before_external_group_by; + extern const SettingsUInt64 max_bytes_in_distinct; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_size_to_preallocate_for_aggregation; + extern const SettingsUInt64 max_subquery_depth; + extern const SettingsUInt64 max_rows_in_distinct; + extern const SettingsUInt64 max_rows_to_group_by; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_count_to_compile_aggregate_expression; + extern const SettingsFloat min_hit_rate_to_use_consecutive_keys_optimization; + extern const SettingsUInt64 min_free_disk_space_for_temporary_data; + extern const SettingsBool optimize_distinct_in_order; + extern const SettingsBool optimize_group_by_constant_keys; + extern const SettingsBool optimize_sorting_by_input_stream_properties; + extern const SettingsBool parallel_replicas_allow_in_with_subquery; + extern const SettingsString parallel_replicas_custom_key; + extern const SettingsUInt64 parallel_replicas_min_number_of_rows_per_replica; + extern const SettingsBool query_plan_enable_multithreading_after_window_functions; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; + extern const SettingsFloat totals_auto_threshold; + extern const SettingsTotalsMode totals_mode; + extern const SettingsBool use_with_fill_by_sorting_prefix; +} + namespace ErrorCodes { @@ -113,7 +157,7 @@ namespace void checkStoragesSupportTransactions(const PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); - if (!query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (!query_context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) return; if (!query_context->getCurrentTransaction()) @@ -158,7 +202,7 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr & const auto & settings = query_context->getSettingsRef(); bool parallel_replicas_estimation_enabled - = query_context->canUseParallelReplicasOnInitiator() && settings.parallel_replicas_min_number_of_rows_per_replica > 0; + = query_context->canUseParallelReplicasOnInitiator() && settings[Setting::parallel_replicas_min_number_of_rows_per_replica] > 0; for (const auto & table_expression : table_nodes) { @@ -277,15 +321,15 @@ public: const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); - aggregate_overflow_row = query_node.isGroupByWithTotals() && settings.max_rows_to_group_by - && settings.group_by_overflow_mode == OverflowMode::ANY && settings.totals_mode != TotalsMode::AFTER_HAVING_EXCLUSIVE; + aggregate_overflow_row = query_node.isGroupByWithTotals() && settings[Setting::max_rows_to_group_by] + && settings[Setting::group_by_overflow_mode] == OverflowMode::ANY && settings[Setting::totals_mode] != TotalsMode::AFTER_HAVING_EXCLUSIVE; aggregate_final = query_processing_info.getToStage() > QueryProcessingStage::WithMergeableState && !query_node.isGroupByWithTotals() && !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube(); aggregation_with_rollup_or_cube_or_grouping_sets = query_node.isGroupByWithRollup() || query_node.isGroupByWithCube() || query_node.isGroupByWithGroupingSets(); aggregation_should_produce_results_in_order_of_bucket_number = query_processing_info.getToStage() == QueryProcessingStage::WithMergeableState - && (settings.distributed_aggregation_memory_efficient || settings.enable_memory_bound_merging_of_aggregation_results); + && (settings[Setting::distributed_aggregation_memory_efficient] || settings[Setting::enable_memory_bound_merging_of_aggregation_results]); query_has_array_join_in_join_tree = queryHasArrayJoinInJoinTree(query_tree); query_has_with_totals_in_any_subquery_in_join_tree = queryHasWithTotalsInAnySubqueryInJoinTree(query_tree); @@ -377,9 +421,9 @@ Aggregator::Params getAggregatorParams(const PlannerContextPtr & planner_context const auto stats_collecting_params = StatsCollectingParams( calculateCacheKey(select_query_info.query), - settings.collect_hash_table_stats_during_aggregation, + settings[Setting::collect_hash_table_stats_during_aggregation], query_context->getServerSettings().max_entries_for_hash_table_stats, - settings.max_size_to_preallocate_for_aggregation); + settings[Setting::max_size_to_preallocate_for_aggregation]); auto aggregate_descriptions = aggregation_analysis_result.aggregate_descriptions; if (aggregate_descriptions_remove_arguments) @@ -392,24 +436,24 @@ Aggregator::Params getAggregatorParams(const PlannerContextPtr & planner_context aggregation_analysis_result.aggregation_keys, aggregate_descriptions, query_analysis_result.aggregate_overflow_row, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - settings.group_by_two_level_threshold, - settings.group_by_two_level_threshold_bytes, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set - || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && aggregation_analysis_result.aggregation_keys.empty() + settings[Setting::max_rows_to_group_by], + settings[Setting::group_by_overflow_mode], + settings[Setting::group_by_two_level_threshold], + settings[Setting::group_by_two_level_threshold_bytes], + settings[Setting::max_bytes_before_external_group_by], + settings[Setting::empty_result_for_aggregation_by_empty_set] + || (settings[Setting::empty_result_for_aggregation_by_constant_keys_on_empty_set] && aggregation_analysis_result.aggregation_keys.empty() && aggregation_analysis_result.group_by_with_constant_keys), query_context->getTempDataOnDisk(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression, - settings.max_block_size, - settings.enable_software_prefetch_in_aggregation, + settings[Setting::max_threads], + settings[Setting::min_free_disk_space_for_temporary_data], + settings[Setting::compile_aggregate_expressions], + settings[Setting::min_count_to_compile_aggregate_expression], + settings[Setting::max_block_size], + settings[Setting::enable_software_prefetch_in_aggregation], /* only_merge */ false, - settings.optimize_group_by_constant_keys, - settings.min_hit_rate_to_use_consecutive_keys_optimization, + settings[Setting::optimize_group_by_constant_keys], + settings[Setting::min_hit_rate_to_use_consecutive_keys_optimization], stats_collecting_params); return aggregator_params; @@ -438,16 +482,16 @@ void addAggregationStep(QueryPlan & query_plan, SortDescription sort_description_for_merging; SortDescription group_by_sort_description; - if (settings.force_aggregation_in_order) + if (settings[Setting::force_aggregation_in_order]) { group_by_sort_description = getSortDescriptionFromNames(aggregation_analysis_result.aggregation_keys); sort_description_for_merging = group_by_sort_description; } - auto merge_threads = settings.max_threads; - auto temporary_data_merge_threads = settings.aggregation_memory_efficient_merge_threads - ? static_cast(settings.aggregation_memory_efficient_merge_threads) - : static_cast(settings.max_threads); + auto merge_threads = settings[Setting::max_threads]; + auto temporary_data_merge_threads = settings[Setting::aggregation_memory_efficient_merge_threads] + ? static_cast(settings[Setting::aggregation_memory_efficient_merge_threads]) + : static_cast(settings[Setting::max_threads]); bool storage_has_evenly_distributed_read = false; const auto & table_expression_node_to_data = planner_context->getTableExpressionNodeToData(); @@ -467,17 +511,17 @@ void addAggregationStep(QueryPlan & query_plan, aggregator_params, aggregation_analysis_result.grouping_sets_parameters_list, query_analysis_result.aggregate_final, - settings.max_block_size, - settings.aggregation_in_order_max_block_bytes, + settings[Setting::max_block_size], + settings[Setting::aggregation_in_order_max_block_bytes], merge_threads, temporary_data_merge_threads, storage_has_evenly_distributed_read, - settings.group_by_use_nulls, + settings[Setting::group_by_use_nulls], std::move(sort_description_for_merging), std::move(group_by_sort_description), query_analysis_result.aggregation_should_produce_results_in_order_of_bucket_number, - settings.enable_memory_bound_merging_of_aggregation_results, - settings.force_aggregation_in_order); + settings[Setting::enable_memory_bound_merging_of_aggregation_results], + settings[Setting::force_aggregation_in_order]); query_plan.addStep(std::move(aggregating_step)); } @@ -506,12 +550,13 @@ void addMergingAggregatedStep(QueryPlan & query_plan, auto keys = aggregation_analysis_result.aggregation_keys; - Aggregator::Params params(keys, + Aggregator::Params params( + keys, aggregation_analysis_result.aggregate_descriptions, query_analysis_result.aggregate_overflow_row, - settings.max_threads, - settings.max_block_size, - settings.min_hit_rate_to_use_consecutive_keys_optimization); + settings[Setting::max_threads], + settings[Setting::max_block_size], + settings[Setting::min_hit_rate_to_use_consecutive_keys_optimization]); bool is_remote_storage = false; bool parallel_replicas_from_merge_tree = false; @@ -532,14 +577,15 @@ void addMergingAggregatedStep(QueryPlan & query_plan, aggregation_analysis_result.grouping_sets_parameters_list, query_analysis_result.aggregate_final, /// Grouping sets don't work with distributed_aggregation_memory_efficient enabled (#43989) - settings.distributed_aggregation_memory_efficient && (is_remote_storage || parallel_replicas_from_merge_tree) && !query_analysis_result.aggregation_with_rollup_or_cube_or_grouping_sets, - settings.max_threads, - settings.aggregation_memory_efficient_merge_threads, + settings[Setting::distributed_aggregation_memory_efficient] && (is_remote_storage || parallel_replicas_from_merge_tree) + && !query_analysis_result.aggregation_with_rollup_or_cube_or_grouping_sets, + settings[Setting::max_threads], + settings[Setting::aggregation_memory_efficient_merge_threads], query_analysis_result.aggregation_should_produce_results_in_order_of_bucket_number, - settings.max_block_size, - settings.aggregation_in_order_max_block_bytes, + settings[Setting::max_block_size], + settings[Setting::aggregation_in_order_max_block_bytes], std::move(group_by_sort_description), - settings.enable_memory_bound_merging_of_aggregation_results); + settings[Setting::enable_memory_bound_merging_of_aggregation_results]); query_plan.addStep(std::move(merging_aggregated)); } @@ -572,8 +618,8 @@ void addTotalsHavingStep(QueryPlan & query_plan, std::move(actions), having_analysis_result.filter_column_name, having_analysis_result.remove_filter_column, - settings.totals_mode, - settings.totals_auto_threshold, + settings[Setting::totals_mode], + settings[Setting::totals_auto_threshold], need_finalize); if (having_analysis_result.filter_actions) @@ -604,13 +650,13 @@ void addCubeOrRollupStepIfNeeded(QueryPlan & query_plan, if (query_node.isGroupByWithRollup()) { auto rollup_step = std::make_unique( - query_plan.getCurrentDataStream(), std::move(aggregator_params), true /*final*/, settings.group_by_use_nulls); + query_plan.getCurrentDataStream(), std::move(aggregator_params), true /*final*/, settings[Setting::group_by_use_nulls]); query_plan.addStep(std::move(rollup_step)); } else if (query_node.isGroupByWithCube()) { auto cube_step = std::make_unique( - query_plan.getCurrentDataStream(), std::move(aggregator_params), true /*final*/, settings.group_by_use_nulls); + query_plan.getCurrentDataStream(), std::move(aggregator_params), true /*final*/, settings[Setting::group_by_use_nulls]); query_plan.addStep(std::move(cube_step)); } } @@ -641,7 +687,7 @@ void addDistinctStep(QueryPlan & query_plan, limit_hint_for_distinct = limit_length + limit_offset; } - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + SizeLimits limits(settings[Setting::max_rows_in_distinct], settings[Setting::max_bytes_in_distinct], settings[Setting::distinct_overflow_mode]); auto distinct_step = std::make_unique( query_plan.getCurrentDataStream(), @@ -649,7 +695,7 @@ void addDistinctStep(QueryPlan & query_plan, limit_hint_for_distinct, column_names, pre_distinct, - settings.optimize_distinct_in_order); + settings[Setting::optimize_distinct_in_order]); distinct_step->setStepDescription(pre_distinct ? "Preliminary DISTINCT" : "DISTINCT"); query_plan.addStep(std::move(distinct_step)); @@ -669,7 +715,7 @@ void addSortingStep(QueryPlan & query_plan, sort_description, query_analysis_result.partial_sorting_limit, sort_settings, - settings.optimize_sorting_by_input_stream_properties); + settings[Setting::optimize_sorting_by_input_stream_properties]); sorting_step->setStepDescription("Sorting for ORDER BY"); query_plan.addStep(std::move(sorting_step)); } @@ -683,13 +729,13 @@ void addMergeSortingStep(QueryPlan & query_plan, const auto & settings = query_context->getSettingsRef(); const auto & sort_description = query_analysis_result.sort_description; - const auto max_block_size = settings.max_block_size; - auto merging_sorted = std::make_unique(query_plan.getCurrentDataStream(), + auto merging_sorted = std::make_unique( + query_plan.getCurrentDataStream(), sort_description, - max_block_size, + settings[Setting::max_block_size], query_analysis_result.partial_sorting_limit, - settings.exact_rows_before_limit); + settings[Setting::exact_rows_before_limit]); merging_sorted->setStepDescription("Merge sorted streams " + description); query_plan.addStep(std::move(merging_sorted)); } @@ -810,7 +856,7 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan, sort_description, std::move(fill_description), interpolate_description, - settings.use_with_fill_by_sorting_prefix); + settings[Setting::use_with_fill_by_sorting_prefix]); query_plan.addStep(std::move(filling_step)); } @@ -855,7 +901,8 @@ void addPreliminaryLimitStep(QueryPlan & query_plan, const auto & query_context = planner_context->getQueryContext(); const Settings & settings = query_context->getSettingsRef(); - auto limit = std::make_unique(query_plan.getCurrentDataStream(), limit_length, limit_offset, settings.exact_rows_before_limit); + auto limit + = std::make_unique(query_plan.getCurrentDataStream(), limit_length, limit_offset, settings[Setting::exact_rows_before_limit]); limit->setStepDescription(do_not_skip_offset ? "preliminary LIMIT (with OFFSET)" : "preliminary LIMIT (without OFFSET)"); query_plan.addStep(std::move(limit)); } @@ -883,16 +930,10 @@ bool addPreliminaryLimitOptimizationStepIfNeeded(QueryPlan & query_plan, } bool apply_limit = query_processing_info.getToStage() != QueryProcessingStage::WithMergeableStateAfterAggregation; - bool apply_prelimit = apply_limit && - query_node.hasLimit() && - !query_node.isLimitWithTies() && - !query_node.isGroupByWithTotals() && - !query_analysis_result.query_has_with_totals_in_any_subquery_in_join_tree && - !query_analysis_result.query_has_array_join_in_join_tree && - !query_node.isDistinct() && - !query_node.hasLimitBy() && - !settings.extremes && - !has_withfill; + bool apply_prelimit = apply_limit && query_node.hasLimit() && !query_node.isLimitWithTies() && !query_node.isGroupByWithTotals() + && !query_analysis_result.query_has_with_totals_in_any_subquery_in_join_tree + && !query_analysis_result.query_has_array_join_in_join_tree && !query_node.isDistinct() && !query_node.hasLimitBy() + && !settings[Setting::extremes] && !has_withfill; bool apply_offset = query_processing_info.getToStage() != QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; if (apply_prelimit) { @@ -981,7 +1022,7 @@ void addWindowSteps(QueryPlan & query_plan, if (need_sort && i != 0) { need_sort = !sortDescriptionIsPrefix(window_description.full_sort_description, window_descriptions[i - 1].full_sort_description) - || (settings.max_threads != 1 && window_description.partition_by.size() != window_descriptions[i - 1].partition_by.size()); + || (settings[Setting::max_threads] != 1 && window_description.partition_by.size() != window_descriptions[i - 1].partition_by.size()); } if (need_sort) { @@ -993,14 +1034,15 @@ void addWindowSteps(QueryPlan & query_plan, window_description.partition_by, 0 /*limit*/, sort_settings, - settings.optimize_sorting_by_input_stream_properties); + settings[Setting::optimize_sorting_by_input_stream_properties]); sorting_step->setStepDescription("Sorting for window '" + window_description.window_name + "'"); query_plan.addStep(std::move(sorting_step)); } // Fan out streams only for the last window to preserve the ordering between windows, // and WindowTransform works on single stream anyway. - const bool streams_fan_out = settings.query_plan_enable_multithreading_after_window_functions && ((i + 1) == window_descriptions_size); + const bool streams_fan_out + = settings[Setting::query_plan_enable_multithreading_after_window_functions] && ((i + 1) == window_descriptions_size); auto window_step = std::make_unique(query_plan.getCurrentDataStream(), window_description, window_description.window_functions, streams_fan_out); @@ -1016,7 +1058,7 @@ void addLimitStep(QueryPlan & query_plan, { const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); - bool always_read_till_end = settings.exact_rows_before_limit; + bool always_read_till_end = settings[Setting::exact_rows_before_limit]; bool limit_with_ties = query_node.isLimitWithTies(); /** Special cases: @@ -1065,7 +1107,7 @@ void addLimitStep(QueryPlan & query_plan, void addExtremesStepIfNeeded(QueryPlan & query_plan, const PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); - if (!query_context->getSettingsRef().extremes) + if (!query_context->getSettingsRef()[Setting::extremes]) return; auto extremes_step = std::make_unique(query_plan.getCurrentDataStream()); @@ -1178,11 +1220,9 @@ PlannerContextPtr buildPlannerContext(const QueryTreeNodePtr & query_tree_node, query_tree_node->formatASTForErrorMessage()); auto & mutable_context = query_node ? query_node->getMutableContext() : union_node->getMutableContext(); - size_t max_subquery_depth = mutable_context->getSettingsRef().max_subquery_depth; + size_t max_subquery_depth = mutable_context->getSettingsRef()[Setting::max_subquery_depth]; if (max_subquery_depth && select_query_options.subquery_depth > max_subquery_depth) - throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, - "Too deep subqueries. Maximum: {}", - max_subquery_depth); + throw Exception(ErrorCodes::TOO_DEEP_SUBQUERIES, "Too deep subqueries. Maximum: {}", max_subquery_depth); const auto & client_info = mutable_context->getClientInfo(); auto min_major = static_cast(DBMS_MIN_MAJOR_VERSION_WITH_CURRENT_AGGREGATION_VARIANT_SELECTION_METHOD); @@ -1325,7 +1365,7 @@ void Planner::buildPlanForUnionNode() const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); - auto max_threads = settings.max_threads; + auto max_threads = settings[Setting::max_threads]; bool is_distinct = union_mode == SelectUnionMode::UNION_DISTINCT || union_mode == SelectUnionMode::INTERSECT_DISTINCT || union_mode == SelectUnionMode::EXCEPT_DISTINCT; @@ -1357,7 +1397,7 @@ void Planner::buildPlanForUnionNode() if (is_distinct) { /// Add distinct transform - SizeLimits limits(settings.max_rows_in_distinct, settings.max_bytes_in_distinct, settings.distinct_overflow_mode); + SizeLimits limits(settings[Setting::max_rows_in_distinct], settings[Setting::max_bytes_in_distinct], settings[Setting::distinct_overflow_mode]); auto distinct_step = std::make_unique( query_plan.getCurrentDataStream(), @@ -1365,7 +1405,7 @@ void Planner::buildPlanForUnionNode() 0 /*limit hint*/, query_plan.getCurrentDataStream().header.getNames(), false /*pre distinct*/, - settings.optimize_distinct_in_order); + settings[Setting::optimize_distinct_in_order]); query_plan.addStep(std::move(distinct_step)); } } @@ -1422,9 +1462,9 @@ void Planner::buildPlanForQueryNode() const auto & settings = query_context->getSettingsRef(); if (query_context->canUseTaskBasedParallelReplicas()) { - if (!settings.parallel_replicas_allow_in_with_subquery && planner_context->getPreparedSets().hasSubqueries()) + if (!settings[Setting::parallel_replicas_allow_in_with_subquery] && planner_context->getPreparedSets().hasSubqueries()) { - if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas"); auto & mutable_context = planner_context->getMutableQueryContext(); @@ -1463,7 +1503,7 @@ void Planner::buildPlanForQueryNode() const auto & modifiers = table_node->getTableExpressionModifiers(); if (modifiers.has_value() && modifiers->hasFinal()) { - if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "FINAL modifier is not supported with parallel replicas"); else { @@ -1477,12 +1517,12 @@ void Planner::buildPlanForQueryNode() } } - if (!settings.parallel_replicas_custom_key.value.empty()) + if (!settings[Setting::parallel_replicas_custom_key].value.empty()) { /// Check support for JOIN for parallel replicas with custom key if (planner_context->getTableExpressionNodeToData().size() > 1) { - if (settings.allow_experimental_parallel_reading_from_replicas >= 2) + if (settings[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JOINs are not supported with parallel replicas"); else { diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 43177fc73c0..80139a713ff 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -36,6 +36,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool transform_null_in; +} namespace ErrorCodes { @@ -818,7 +822,8 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma if (left_tuple_type && left_tuple_type->getElements().size() != 1) set_element_types = left_tuple_type->getElements(); - set_element_types = Set::getElementTypes(std::move(set_element_types), planner_context->getQueryContext()->getSettingsRef().transform_null_in); + set_element_types + = Set::getElementTypes(std::move(set_element_types), planner_context->getQueryContext()->getSettingsRef()[Setting::transform_null_in]); set = planner_context->getPreparedSets().findTuple(set_key, set_element_types); } else diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index ed3f78193ee..2bb494900f9 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -27,6 +27,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool group_by_use_nulls; +} namespace ErrorCodes { @@ -132,8 +136,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP PlannerActionsVisitor actions_visitor(planner_context); /// Add expressions from GROUP BY - bool group_by_use_nulls = planner_context->getQueryContext()->getSettingsRef().group_by_use_nulls && - (query_node.isGroupByWithGroupingSets() || query_node.isGroupByWithRollup() || query_node.isGroupByWithCube()); + bool group_by_use_nulls = planner_context->getQueryContext()->getSettingsRef()[Setting::group_by_use_nulls] + && (query_node.isGroupByWithGroupingSets() || query_node.isGroupByWithRollup() || query_node.isGroupByWithCube()); bool is_secondary_query = planner_context->getQueryContext()->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; bool is_distributed_query = planner_context->getQueryContext()->isDistributed(); @@ -219,6 +223,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; auto column_after_aggregation = group_by_use_nulls && expression_dag_node->column != nullptr ? makeNullableSafe(expression_dag_node->column) : expression_dag_node->column; + available_columns_after_aggregation.emplace_back(std::move(column_after_aggregation), expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->dag.getOutputs().push_back(expression_dag_node); diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index e0668a0eb8e..10ae2a42e30 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -71,6 +71,39 @@ namespace DB { +namespace Setting +{ + extern const SettingsMap additional_table_filters; + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; + extern const SettingsBool allow_experimental_query_deduplication; + extern const SettingsBool async_socket_for_remote; + extern const SettingsBool empty_result_for_aggregation_by_empty_set; + extern const SettingsBool enable_unaligned_array_join; + extern const SettingsBool join_use_nulls; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_columns_to_read; + extern const SettingsUInt64 max_distributed_connections; + extern const SettingsUInt64 max_rows_in_set_to_optimize_join; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsFloat max_streams_to_max_threads_ratio; + extern const SettingsMaxThreads max_threads; + extern const SettingsBool optimize_sorting_by_input_stream_properties; + extern const SettingsBool optimize_trivial_count_query; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsString parallel_replicas_custom_key; + extern const SettingsParallelReplicasCustomKeyFilterType parallel_replicas_custom_key_filter_type; + extern const SettingsUInt64 parallel_replicas_custom_key_range_lower; + extern const SettingsUInt64 parallel_replicas_custom_key_range_upper; + extern const SettingsBool parallel_replicas_for_non_replicated_merge_tree; + extern const SettingsUInt64 parallel_replicas_min_number_of_rows_per_replica; + extern const SettingsUInt64 parallel_replica_offset; + extern const SettingsBool optimize_move_to_prewhere; + extern const SettingsBool optimize_move_to_prewhere_if_final; + extern const SettingsBool use_concurrency_control; +} namespace ErrorCodes { @@ -224,7 +257,7 @@ bool applyTrivialCountIfPossible( const Names & columns_names) { const auto & settings = query_context->getSettingsRef(); - if (!settings.optimize_trivial_count_query) + if (!settings[Setting::optimize_trivial_count_query]) return false; const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); @@ -266,8 +299,7 @@ bool applyTrivialCountIfPossible( if (main_query_node.hasGroupBy() || main_query_node.hasPrewhere() || main_query_node.hasWhere()) return false; - if (settings.allow_experimental_query_deduplication - || settings.empty_result_for_aggregation_by_empty_set) + if (settings[Setting::allow_experimental_query_deduplication] || settings[Setting::empty_result_for_aggregation_by_empty_set]) return false; QueryTreeNodes aggregates = collectAggregateFunctionNodes(query_tree); @@ -290,9 +322,9 @@ bool applyTrivialCountIfPossible( if (!num_rows) return false; - if (settings.max_parallel_replicas > 1) + if (settings[Setting::max_parallel_replicas] > 1) { - if (!settings.parallel_replicas_custom_key.value.empty() || settings.allow_experimental_parallel_reading_from_replicas == 0) + if (!settings[Setting::parallel_replicas_custom_key].value.empty() || settings[Setting::allow_experimental_parallel_reading_from_replicas] == 0) return false; /// The query could use trivial count if it didn't use parallel replicas, so let's disable it @@ -388,11 +420,12 @@ void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expr } /// Limitation on the number of columns to read - if (settings.max_columns_to_read && columns_names.size() > settings.max_columns_to_read) - throw Exception(ErrorCodes::TOO_MANY_COLUMNS, + if (settings[Setting::max_columns_to_read] && columns_names.size() > settings[Setting::max_columns_to_read]) + throw Exception( + ErrorCodes::TOO_MANY_COLUMNS, "Limit for number of columns to read exceeded. Requested: {}, maximum: {}", columns_names.size(), - settings.max_columns_to_read); + settings[Setting::max_columns_to_read]); } void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info, @@ -411,8 +444,8 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info auto & table_expression_modifiers = table_expression_query_info.table_expression_modifiers; if (table_expression_modifiers) { - if (table_expression_modifiers->hasSampleSizeRatio() || - table_expression_query_info.planner_context->getQueryContext()->getSettingsRef().parallel_replicas_count > 1) + if (table_expression_modifiers->hasSampleSizeRatio() + || table_expression_query_info.planner_context->getQueryContext()->getSettingsRef()[Setting::parallel_replicas_count] > 1) { /// We evaluate sampling for Merge lazily so we need to get all the columns if (storage_snapshot->storage.getName() == "Merge") @@ -487,10 +520,10 @@ std::optional buildCustomKeyFilterIfNeeded(const StoragePtr & sto const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); - if (settings.parallel_replicas_count <= 1 || settings.parallel_replicas_custom_key.value.empty()) + if (settings[Setting::parallel_replicas_count] <= 1 || settings[Setting::parallel_replicas_custom_key].value.empty()) return {}; - auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *query_context); + auto custom_key_ast = parseCustomKeyForTable(settings[Setting::parallel_replicas_custom_key], *query_context); if (!custom_key_ast) throw DB::Exception( ErrorCodes::BAD_ARGUMENTS, @@ -498,15 +531,15 @@ std::optional buildCustomKeyFilterIfNeeded(const StoragePtr & sto "(setting 'max_parallel_replicas'), but the table does not have custom_key defined for it " " or it's invalid (setting 'parallel_replicas_custom_key')"); - LOG_TRACE(getLogger("Planner"), "Processing query on a replica using custom_key '{}'", settings.parallel_replicas_custom_key.value); + LOG_TRACE(getLogger("Planner"), "Processing query on a replica using custom_key '{}'", settings[Setting::parallel_replicas_custom_key].value); auto parallel_replicas_custom_filter_ast = getCustomKeyFilterForParallelReplica( - settings.parallel_replicas_count, - settings.parallel_replica_offset, + settings[Setting::parallel_replicas_count], + settings[Setting::parallel_replica_offset], std::move(custom_key_ast), - {settings.parallel_replicas_custom_key_filter_type, - settings.parallel_replicas_custom_key_range_lower, - settings.parallel_replicas_custom_key_range_upper}, + {settings[Setting::parallel_replicas_custom_key_filter_type], + settings[Setting::parallel_replicas_custom_key_range_lower], + settings[Setting::parallel_replicas_custom_key_range_upper]}, storage->getInMemoryMetadataPtr()->columns, query_context); @@ -522,7 +555,7 @@ std::optional buildAdditionalFiltersIfNeeded(const StoragePtr & s const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); - auto const & additional_filters = settings.additional_table_filters.value; + auto const & additional_filters = settings[Setting::additional_table_filters].value; if (additional_filters.empty()) return {}; @@ -541,8 +574,13 @@ std::optional buildAdditionalFiltersIfNeeded(const StoragePtr & s { ParserExpression parser; additional_filter_ast = parseQuery( - parser, filter.data(), filter.data() + filter.size(), - "additional filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parser, + filter.data(), + filter.data() + filter.size(), + "additional filter", + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); break; } } @@ -652,8 +690,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres table_expression_query_info.current_table_chosen_for_reading_with_parallel_replicas = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table; - size_t max_streams = settings.max_threads; - size_t max_threads_execute_query = settings.max_threads; + size_t max_streams = settings[Setting::max_threads]; + size_t max_threads_execute_query = settings[Setting::max_threads]; /** * To simultaneously query more remote servers when async_socket_for_remote is off @@ -667,14 +705,14 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres * threads are not blocked waiting for data from remote servers. * */ - bool is_sync_remote = table_expression_data.isRemote() && !settings.async_socket_for_remote; + bool is_sync_remote = table_expression_data.isRemote() && !settings[Setting::async_socket_for_remote]; if (is_sync_remote) { - max_streams = settings.max_distributed_connections; - max_threads_execute_query = settings.max_distributed_connections; + max_streams = settings[Setting::max_distributed_connections]; + max_threads_execute_query = settings[Setting::max_distributed_connections]; } - UInt64 max_block_size = settings.max_block_size; + UInt64 max_block_size = settings[Setting::max_block_size]; UInt64 max_block_size_limited = 0; if (is_single_table_expression) { @@ -718,7 +756,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres /// If necessary, we request more sources than the number of threads - to distribute the work evenly over the threads if (max_streams > 1 && !is_sync_remote) { - if (auto streams_with_ratio = max_streams * settings.max_streams_to_max_threads_ratio; canConvertTo(streams_with_ratio)) + if (auto streams_with_ratio = max_streams * settings[Setting::max_streams_to_max_threads_ratio]; + canConvertTo(streams_with_ratio)) max_streams = static_cast(streams_with_ratio); else throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, @@ -755,7 +794,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres /// Apply trivial_count optimization if possible bool is_trivial_count_applied = !select_query_options.only_analyze && is_single_table_expression - && (table_node || table_function_node) && select_query_info.has_aggregates && settings.additional_table_filters.value.empty() + && (table_node || table_function_node) && select_query_info.has_aggregates && settings[Setting::additional_table_filters].value.empty() && applyTrivialCountIfPossible( query_plan, table_expression_query_info, @@ -795,10 +834,11 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres bool is_final = table_expression_query_info.table_expression_modifiers && table_expression_query_info.table_expression_modifiers->hasFinal(); bool optimize_move_to_prewhere - = settings.optimize_move_to_prewhere && (!is_final || settings.optimize_move_to_prewhere_if_final); + = settings[Setting::optimize_move_to_prewhere] && (!is_final || settings[Setting::optimize_move_to_prewhere_if_final]); auto supported_prewhere_columns = storage->supportedPrewhereColumns(); - if (storage->canMoveConditionsToPrewhere() && optimize_move_to_prewhere && (!supported_prewhere_columns || supported_prewhere_columns->contains(filter_info.column_name))) + if (storage->canMoveConditionsToPrewhere() && optimize_move_to_prewhere + && (!supported_prewhere_columns || supported_prewhere_columns->contains(filter_info.column_name))) { if (!prewhere_info) { @@ -836,7 +876,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (query_context->canUseParallelReplicasCustomKey()) { - if (settings.parallel_replicas_count > 1) + if (settings[Setting::parallel_replicas_count] > 1) { if (auto parallel_replicas_custom_key_filter_info= buildCustomKeyFilterIfNeeded(storage, table_expression_query_info, planner_context)) add_filter(*parallel_replicas_custom_key_filter_info, "Parallel replicas custom key filter"); @@ -884,7 +924,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (!table->isMergeTree()) return false; - if (!table->supportsReplication() && !query_settings.parallel_replicas_for_non_replicated_merge_tree) + if (!table->supportsReplication() && !query_settings[Setting::parallel_replicas_for_non_replicated_merge_tree]) return false; return true; @@ -944,7 +984,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres chassert(reading); // (2) if it's ReadFromMergeTree - run index analysis and check number of rows to read - if (settings.parallel_replicas_min_number_of_rows_per_replica > 0) + if (settings[Setting::parallel_replicas_min_number_of_rows_per_replica] > 0) { auto result_ptr = reading->selectRangesToRead(); UInt64 rows_to_read = result_ptr->selected_rows; @@ -958,7 +998,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres rows_to_read = max_block_size_limited; const size_t number_of_replicas_to_use - = rows_to_read / settings.parallel_replicas_min_number_of_rows_per_replica; + = rows_to_read / settings[Setting::parallel_replicas_min_number_of_rows_per_replica]; LOG_TRACE( getLogger("Planner"), "Estimated {} rows to read. It is enough work for {} parallel replicas", @@ -972,7 +1012,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", UInt64{1}); LOG_DEBUG(getLogger("Planner"), "Disabling parallel replicas because there aren't enough rows to read"); } - else if (number_of_replicas_to_use < settings.max_parallel_replicas) + else if (number_of_replicas_to_use < settings[Setting::max_parallel_replicas]) { planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", number_of_replicas_to_use); LOG_DEBUG(getLogger("Planner"), "Reducing the number of replicas to use to {}", number_of_replicas_to_use); @@ -1056,7 +1096,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (!query_plan.getMaxThreads() || is_sync_remote) query_plan.setMaxThreads(max_threads_execute_query); - query_plan.setConcurrencyControl(settings.use_concurrency_control); + query_plan.setConcurrencyControl(settings[Setting::use_concurrency_control]); } else { @@ -1307,7 +1347,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ const auto & query_context = planner_context->getQueryContext(); const auto & settings = query_context->getSettingsRef(); - if (settings.join_use_nulls) + if (settings[Setting::join_use_nulls]) { auto to_nullable_function = FunctionFactory::instance().get("toNullable", query_context); if (isFull(join_kind)) @@ -1486,10 +1526,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ bool is_filled_join = join_algorithm->isFilled(); if (is_filled_join) { - auto filled_join_step = std::make_unique( - left_plan.getCurrentDataStream(), - join_algorithm, - settings.max_block_size); + auto filled_join_step + = std::make_unique(left_plan.getCurrentDataStream(), join_algorithm, settings[Setting::max_block_size]); filled_join_step->setStepDescription("Filled JOIN"); left_plan.addStep(std::move(filled_join_step)); @@ -1512,7 +1550,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ std::move(sort_description), 0 /*limit*/, sort_settings, - settings.optimize_sorting_by_input_stream_properties); + settings[Setting::optimize_sorting_by_input_stream_properties]); sorting_step->setStepDescription(fmt::format("Sort {} before JOIN", join_table_side)); plan.addStep(std::move(sorting_step)); }; @@ -1521,11 +1559,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ auto add_create_set = [&settings, crosswise_connection](QueryPlan & plan, const Names & key_names, JoinTableSide join_table_side) { auto creating_set_step = std::make_unique( - plan.getCurrentDataStream(), - key_names, - settings.max_rows_in_set_to_optimize_join, - crosswise_connection, - join_table_side); + plan.getCurrentDataStream(), key_names, settings[Setting::max_rows_in_set_to_optimize_join], crosswise_connection, join_table_side); creating_set_step->setStepDescription(fmt::format("Create set and filter {} joined stream", join_table_side)); auto * step_raw_ptr = creating_set_step.get(); @@ -1559,7 +1593,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ bool has_non_const_keys = has_non_const(left_plan.getCurrentDataStream().header, join_clause.key_names_left) && has_non_const(right_plan.getCurrentDataStream().header, join_clause.key_names_right); - if (settings.max_rows_in_set_to_optimize_join > 0 && join_type_allows_filtering && has_non_const_keys) + if (settings[Setting::max_rows_in_set_to_optimize_join] > 0 && join_type_allows_filtering && has_non_const_keys) { auto * left_set = add_create_set(left_plan, join_clause.key_names_left, JoinTableSide::Left); auto * right_set = add_create_set(right_plan, join_clause.key_names_right, JoinTableSide::Right); @@ -1580,8 +1614,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_ left_plan.getCurrentDataStream(), right_plan.getCurrentDataStream(), std::move(join_algorithm), - settings.max_block_size, - settings.max_threads, + settings[Setting::max_block_size], + settings[Setting::max_threads], false /*optimize_read_in_order*/); join_step->setStepDescription(fmt::format("JOIN {}", join_pipeline_type)); @@ -1732,8 +1766,8 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_ auto array_join_step = std::make_unique( plan.getCurrentDataStream(), ArrayJoin{std::move(array_join_column_names), array_join_node.isLeft()}, - settings.enable_unaligned_array_join, - settings.max_block_size); + settings[Setting::enable_unaligned_array_join], + settings[Setting::max_block_size]); array_join_step->setStepDescription("ARRAY JOIN"); plan.addStep(std::move(array_join_step)); diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 42aeb2b9b1d..8857fda5b4c 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -1,8 +1,5 @@ #include -#include -#include - #include #include #include @@ -48,6 +45,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_join_condition; + extern const SettingsBool collect_hash_table_stats_during_joins; + extern const SettingsBool join_any_take_last_row; + extern const SettingsBool join_use_nulls; + extern const SettingsUInt64 max_size_to_preallocate_for_joins; + extern const SettingsMaxThreads max_threads; +} namespace ErrorCodes { @@ -317,8 +323,8 @@ void buildJoinClause( } else { - auto support_mixed_join_condition = planner_context->getQueryContext()->getSettingsRef().allow_experimental_join_condition; - auto join_use_nulls = planner_context->getQueryContext()->getSettingsRef().join_use_nulls; + auto support_mixed_join_condition = planner_context->getQueryContext()->getSettingsRef()[Setting::allow_experimental_join_condition]; + auto join_use_nulls = planner_context->getQueryContext()->getSettingsRef()[Setting::join_use_nulls]; /// If join_use_nulls = true, the columns' nullability will be changed later which make this expression not right. if (support_mixed_join_condition && !join_use_nulls) { @@ -353,8 +359,8 @@ void buildJoinClause( } else { - auto support_mixed_join_condition = planner_context->getQueryContext()->getSettingsRef().allow_experimental_join_condition; - auto join_use_nulls = planner_context->getQueryContext()->getSettingsRef().join_use_nulls; + auto support_mixed_join_condition = planner_context->getQueryContext()->getSettingsRef()[Setting::allow_experimental_join_condition]; + auto join_use_nulls = planner_context->getQueryContext()->getSettingsRef()[Setting::join_use_nulls]; /// If join_use_nulls = true, the columns' nullability will be changed later which make this expression not right. if (support_mixed_join_condition && !join_use_nulls) { @@ -574,7 +580,7 @@ JoinClausesAndActions buildJoinClausesAndActions( if (join_clause.isNullsafeCompareKey(i) && left_key_node->result_type->isNullable() && right_key_node->result_type->isNullable()) { /** - * In case of null-safe comparison (a IS NOT DISTICT FROM b), + * In case of null-safe comparison (a IS NOT DISTINCT FROM b), * we need to wrap keys with a non-nullable type. * The type `tuple` can be used for this purpose, * because value tuple(NULL) is not NULL itself (moreover it has type Tuple(Nullable(T) which is not Nullable). @@ -815,14 +821,15 @@ static std::shared_ptr tryCreateJoin(JoinAlgorithm algorithm, const auto & settings = query_context->getSettingsRef(); StatsCollectingParams params{ calculateCacheKey(table_join, right_table_expression), - settings.collect_hash_table_stats_during_joins, + settings[Setting::collect_hash_table_stats_during_joins], query_context->getServerSettings().max_entries_for_hash_table_stats, - settings.max_size_to_preallocate_for_joins}; + settings[Setting::max_size_to_preallocate_for_joins]}; return std::make_shared( - query_context, table_join, query_context->getSettingsRef().max_threads, right_table_expression_header, params); + query_context, table_join, query_context->getSettingsRef()[Setting::max_threads], right_table_expression_header, params); } - return std::make_shared(table_join, right_table_expression_header, query_context->getSettingsRef().join_any_take_last_row); + return std::make_shared( + table_join, right_table_expression_header, query_context->getSettingsRef()[Setting::join_any_take_last_row]); } if (algorithm == JoinAlgorithm::FULL_SORTING_MERGE) diff --git a/src/Planner/PlannerSorting.cpp b/src/Planner/PlannerSorting.cpp index 1c6e820fed1..af51afdef13 100644 --- a/src/Planner/PlannerSorting.cpp +++ b/src/Planner/PlannerSorting.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool compile_sort_description; + extern const SettingsUInt64 min_count_to_compile_sort_description; +} namespace ErrorCodes { @@ -150,8 +155,8 @@ SortDescription extractSortDescription(const QueryTreeNodePtr & order_by_node, c } const auto & settings = planner_context.getQueryContext()->getSettingsRef(); - sort_column_description.compile_sort_description = settings.compile_sort_description; - sort_column_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description; + sort_column_description.compile_sort_description = settings[Setting::compile_sort_description]; + sort_column_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description]; return sort_column_description; } diff --git a/src/Planner/PlannerWindowFunctions.cpp b/src/Planner/PlannerWindowFunctions.cpp index f91cf644cf0..ce27c4ae86e 100644 --- a/src/Planner/PlannerWindowFunctions.cpp +++ b/src/Planner/PlannerWindowFunctions.cpp @@ -16,6 +16,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool compile_sort_description; + extern const SettingsUInt64 min_count_to_compile_sort_description; +} namespace ErrorCodes { @@ -56,8 +61,8 @@ WindowDescription extractWindowDescriptionFromWindowNode(const QueryTreeNodePtr const auto & query_context = planner_context.getQueryContext(); const auto & query_context_settings = query_context->getSettingsRef(); - bool compile_sort_description = query_context_settings.compile_sort_description; - size_t min_count_to_compile_sort_description = query_context_settings.min_count_to_compile_sort_description; + bool compile_sort_description = query_context_settings[Setting::compile_sort_description]; + size_t min_count_to_compile_sort_description = query_context_settings[Setting::min_count_to_compile_sort_description]; window_description.partition_by.compile_sort_description = compile_sort_description; window_description.partition_by.min_count_to_compile_sort_description = min_count_to_compile_sort_description; diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index 822a3e9465e..35b99380ebf 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -48,6 +48,27 @@ namespace DB { +namespace Setting +{ + extern const SettingsString additional_result_filter; + extern const SettingsUInt64 max_bytes_to_read; + extern const SettingsUInt64 max_bytes_to_read_leaf; + extern const SettingsSeconds max_estimated_execution_time; + extern const SettingsUInt64 max_execution_speed; + extern const SettingsUInt64 max_execution_speed_bytes; + extern const SettingsSeconds max_execution_time; + extern const SettingsUInt64 max_query_size; + extern const SettingsUInt64 max_rows_to_read; + extern const SettingsUInt64 max_rows_to_read_leaf; + extern const SettingsUInt64 min_execution_speed; + extern const SettingsUInt64 min_execution_speed_bytes; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsOverflowMode read_overflow_mode; + extern const SettingsOverflowMode read_overflow_mode_leaf; + extern const SettingsSeconds timeout_before_checking_execution_speed; + extern const SettingsOverflowMode timeout_overflow_mode; +} namespace ErrorCodes { @@ -174,9 +195,9 @@ StreamLocalLimits getLimitsForStorage(const Settings & settings, const SelectQue { StreamLocalLimits limits; limits.mode = LimitsMode::LIMITS_TOTAL; - limits.size_limits = SizeLimits(settings.max_rows_to_read, settings.max_bytes_to_read, settings.read_overflow_mode); - limits.speed_limits.max_execution_time = settings.max_execution_time; - limits.timeout_overflow_mode = settings.timeout_overflow_mode; + limits.size_limits = SizeLimits(settings[Setting::max_rows_to_read], settings[Setting::max_bytes_to_read], settings[Setting::read_overflow_mode]); + limits.speed_limits.max_execution_time = settings[Setting::max_execution_time]; + limits.timeout_overflow_mode = settings[Setting::timeout_overflow_mode]; /** Quota and minimal speed restrictions are checked on the initiating server of the request, and not on remote servers, * because the initiating server has a summary of the execution of the request on all servers. @@ -189,14 +210,14 @@ StreamLocalLimits getLimitsForStorage(const Settings & settings, const SelectQue */ if (options.to_stage == QueryProcessingStage::Complete) { - limits.speed_limits.min_execution_rps = settings.min_execution_speed; - limits.speed_limits.min_execution_bps = settings.min_execution_speed_bytes; + limits.speed_limits.min_execution_rps = settings[Setting::min_execution_speed]; + limits.speed_limits.min_execution_bps = settings[Setting::min_execution_speed_bytes]; } - limits.speed_limits.max_execution_rps = settings.max_execution_speed; - limits.speed_limits.max_execution_bps = settings.max_execution_speed_bytes; - limits.speed_limits.timeout_before_checking_execution_speed = settings.timeout_before_checking_execution_speed; - limits.speed_limits.max_estimated_execution_time = settings.max_estimated_execution_time; + limits.speed_limits.max_execution_rps = settings[Setting::max_execution_speed]; + limits.speed_limits.max_execution_bps = settings[Setting::max_execution_speed_bytes]; + limits.speed_limits.timeout_before_checking_execution_speed = settings[Setting::timeout_before_checking_execution_speed]; + limits.speed_limits.max_estimated_execution_time = settings[Setting::max_estimated_execution_time]; return limits; } @@ -214,7 +235,7 @@ StorageLimits buildStorageLimits(const Context & context, const SelectQueryOptio if (!options.ignore_limits) { limits = getLimitsForStorage(settings, options); - leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, settings.max_bytes_to_read_leaf, settings.read_overflow_mode_leaf); + leaf_limits = SizeLimits(settings[Setting::max_rows_to_read_leaf], settings[Setting::max_bytes_to_read_leaf], settings[Setting::read_overflow_mode_leaf]); } return {limits, leaf_limits}; @@ -471,14 +492,19 @@ FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree, ASTPtr parseAdditionalResultFilter(const Settings & settings) { - const String & additional_result_filter = settings.additional_result_filter; + const String & additional_result_filter = settings[Setting::additional_result_filter]; if (additional_result_filter.empty()) return {}; ParserExpression parser; auto additional_result_filter_ast = parseQuery( - parser, additional_result_filter.data(), additional_result_filter.data() + additional_result_filter.size(), - "additional result filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + parser, + additional_result_filter.data(), + additional_result_filter.data() + additional_result_filter.size(), + "additional result filter", + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); return additional_result_filter_ast; } diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp index 25481d06670..f2f2c881af6 100644 --- a/src/Planner/findParallelReplicasQuery.cpp +++ b/src/Planner/findParallelReplicasQuery.cpp @@ -24,6 +24,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool parallel_replicas_allow_in_with_subquery; +} namespace ErrorCodes { @@ -202,7 +206,7 @@ const QueryNode * findQueryForParallelReplicas( const auto * filter = typeid_cast(step); const auto * creating_sets = typeid_cast(step); - bool allowed_creating_sets = settings.parallel_replicas_allow_in_with_subquery && creating_sets; + bool allowed_creating_sets = settings[Setting::parallel_replicas_allow_in_with_subquery] && creating_sets; if (!expression && !filter && !allowed_creating_sets) can_distribute_full_node = false; diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index 72e1afaafaa..f6fab4552e8 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -28,6 +28,11 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool log_processors_profiles; + extern const SettingsBool opentelemetry_trace_processors; +} namespace ErrorCodes { @@ -40,8 +45,8 @@ PipelineExecutor::PipelineExecutor(std::shared_ptr & processors, Que { if (process_list_element) { - profile_processors = process_list_element->getContext()->getSettingsRef().log_processors_profiles; - trace_processors = process_list_element->getContext()->getSettingsRef().opentelemetry_trace_processors; + profile_processors = process_list_element->getContext()->getSettingsRef()[Setting::log_processors_profiles]; + trace_processors = process_list_element->getContext()->getSettingsRef()[Setting::opentelemetry_trace_processors]; } try { diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 6083e1a6bc8..736188ece1b 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include diff --git a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp index 8ab19331b11..1784763022c 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.cpp @@ -2,6 +2,7 @@ #if USE_CAPNP #include +#include #include #include #include diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index 566a036d79c..e04d77996f3 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -32,6 +32,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -543,7 +548,8 @@ bool ConstantExpressionTemplate::parseLiteralAndAssertType( ParserArrayOfLiterals parser_array; ParserTupleOfLiterals parser_tuple; - IParser::Pos iterator(token_iterator, static_cast(settings.max_parser_depth), static_cast(settings.max_parser_backtracks)); + IParser::Pos iterator( + token_iterator, static_cast(settings[Setting::max_parser_depth]), static_cast(settings[Setting::max_parser_backtracks])); while (iterator->begin < istr.position()) ++iterator; Expected expected; diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp index 6d73abfe840..328ad19b377 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp index 7fa3d90ce81..a3e9cad4803 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp index b64318e4093..9a75e4214c3 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.cpp @@ -1,3 +1,4 @@ +#include #include #include diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp index a72c6037619..a0218c29f3a 100644 --- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h index f33108472de..a471ed0eac2 100644 --- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/src/Processors/Formats/Impl/JSONObjectEachRowRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONObjectEachRowRowOutputFormat.cpp index 5d8e74309e3..a6b12ad95f4 100644 --- a/src/Processors/Formats/Impl/JSONObjectEachRowRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONObjectEachRowRowOutputFormat.cpp @@ -1,7 +1,8 @@ -#include -#include +#include #include #include +#include +#include namespace DB { diff --git a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp index 036539c87e7..3ade4c080b0 100644 --- a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp @@ -2,7 +2,7 @@ #include #include #include - +#include namespace DB { diff --git a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h index 3803056123f..7873e973b2f 100644 --- a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h +++ b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h @@ -2,7 +2,6 @@ #include #include -#include #include namespace DB diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp index 00bb5ff6fcf..c79cdaf72fd 100644 --- a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h index 8a4ae1f3b96..7a6b7a3ea7a 100644 --- a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include namespace DB diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 6b7f1f5206c..afd302237de 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -1,3 +1,4 @@ +#include #include #if USE_MSGPACK diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index 76c468b3983..ea136eb7296 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -6,7 +6,6 @@ #include #include -#include #include #include #include diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.h b/src/Processors/Formats/Impl/MySQLOutputFormat.h index 6161b6bdc14..dfaa52878c3 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.h +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.h @@ -1,11 +1,10 @@ #pragma once #include -#include - #include +#include #include - +#include namespace DB { diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 02c74742226..cb6cf00e981 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -7,8 +7,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -27,6 +27,9 @@ namespace CurrentMetrics namespace DB { +class IOutputFormat; +using OutputFormatPtr = std::shared_ptr; + namespace ErrorCodes { extern const int NOT_IMPLEMENTED; diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h index ed528cc077c..a83c172c2ed 100644 --- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h @@ -7,6 +7,8 @@ #include #include +#include + namespace parquet { class FileMetaData; } namespace parquet::arrow { class FileReader; } namespace arrow { class Buffer; class RecordBatchReader;} diff --git a/src/Processors/Formats/Impl/PostgreSQLOutputFormat.cpp b/src/Processors/Formats/Impl/PostgreSQLOutputFormat.cpp index 3fa126494d2..4ac874b883d 100644 --- a/src/Processors/Formats/Impl/PostgreSQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PostgreSQLOutputFormat.cpp @@ -1,5 +1,6 @@ -#include #include "PostgreSQLOutputFormat.h" +#include +#include namespace DB { diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index 213e1c785fd..f98c8120854 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -8,7 +8,6 @@ namespace DB { -class DB; class ProtobufSerializer; class ProtobufWriter; class WriteBuffer; diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp index a5cbb85eecd..6c4422b36b5 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.h b/src/Processors/Formats/Impl/RegexpRowInputFormat.h index 8016593691f..6ad7bde3c96 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.h +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include diff --git a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp index 6f490c8b085..d7acb6bbcaa 100644 --- a/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp @@ -1,5 +1,6 @@ -#include +#include #include +#include namespace DB diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 16b88d0b8dc..444fe1ce83d 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -21,6 +21,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; +} namespace ErrorCodes { @@ -195,7 +200,10 @@ void ValuesBlockInputFormat::readUntilTheEndOfRowAndReTokenize(size_t current_co auto * row_end = buf->position(); buf->rollbackToCheckpoint(); tokens.emplace(buf->position(), row_end); - token_iterator.emplace(*tokens, static_cast(context->getSettingsRef().max_parser_depth), static_cast(context->getSettingsRef().max_parser_backtracks)); + token_iterator.emplace( + *tokens, + static_cast(context->getSettingsRef()[Setting::max_parser_depth]), + static_cast(context->getSettingsRef()[Setting::max_parser_backtracks])); auto const & first = (*token_iterator).get(); if (first.isError() || first.isEnd()) { @@ -419,7 +427,8 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx { Expected expected; /// Keep a copy to the start of the column tokens to use if later if necessary - ti_start = IParser::Pos(*token_iterator, static_cast(settings.max_parser_depth), static_cast(settings.max_parser_backtracks)); + ti_start = IParser::Pos( + *token_iterator, static_cast(settings[Setting::max_parser_depth]), static_cast(settings[Setting::max_parser_backtracks])); parsed = parser.parse(*token_iterator, ast, expected); diff --git a/src/Processors/Formats/InputFormatErrorsLogger.cpp b/src/Processors/Formats/InputFormatErrorsLogger.cpp index b5aa936df07..cd232406a2d 100644 --- a/src/Processors/Formats/InputFormatErrorsLogger.cpp +++ b/src/Processors/Formats/InputFormatErrorsLogger.cpp @@ -6,10 +6,17 @@ #include #include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsString errors_output_format; + extern const SettingsString input_format_record_errors_file_path; + extern const SettingsUInt64 max_block_size; +} namespace ErrorCodes { @@ -21,9 +28,9 @@ namespace const String DEFAULT_OUTPUT_FORMAT = "CSV"; } -InputFormatErrorsLogger::InputFormatErrorsLogger(const ContextPtr & context) : max_block_size(context->getSettingsRef().max_block_size) +InputFormatErrorsLogger::InputFormatErrorsLogger(const ContextPtr & context) : max_block_size(context->getSettingsRef()[Setting::max_block_size]) { - String output_format = context->getSettingsRef().errors_output_format; + String output_format = context->getSettingsRef()[Setting::errors_output_format]; if (!FormatFactory::instance().isOutputFormat(output_format)) output_format = DEFAULT_OUTPUT_FORMAT; if (context->hasInsertionTable()) @@ -31,7 +38,7 @@ InputFormatErrorsLogger::InputFormatErrorsLogger(const ContextPtr & context) : m if (context->getInsertionTable().hasDatabase()) database = context->getInsertionTable().getDatabaseName(); - String path_in_setting = context->getSettingsRef().input_format_record_errors_file_path; + String path_in_setting = context->getSettingsRef()[Setting::input_format_record_errors_file_path]; if (context->getApplicationType() == Context::ApplicationType::SERVER) { diff --git a/src/Processors/Formats/InputFormatErrorsLogger.h b/src/Processors/Formats/InputFormatErrorsLogger.h index 2230490f66a..e9fa073b249 100644 --- a/src/Processors/Formats/InputFormatErrorsLogger.h +++ b/src/Processors/Formats/InputFormatErrorsLogger.h @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { diff --git a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h index cd836cb00dc..dbeb160922a 100644 --- a/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h +++ b/src/Processors/Formats/RowInputFormatWithNamesAndTypes.h @@ -3,7 +3,6 @@ #include #include #include -#include namespace DB { diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index eb699858bdf..15e2ead4e74 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -8,6 +8,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} std::unique_ptr createLocalPlan( const ASTPtr & query_ast, @@ -35,7 +39,7 @@ std::unique_ptr createLocalPlan( .setShardInfo(static_cast(shard_num), static_cast(shard_count)) .ignoreASTOptimizations(); - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { /// For Analyzer, identifier in GROUP BY/ORDER BY/LIMIT BY lists has been resolved to /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index 4d984133efd..3431eab6b55 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -4,52 +4,80 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_aggregate_partitions_independently; + extern const SettingsBool force_optimize_projection; + extern const SettingsString force_optimize_projection_name; + extern const SettingsBool optimize_aggregation_in_order; + extern const SettingsBool optimize_distinct_in_order; + extern const SettingsBool optimize_read_in_order; + extern const SettingsBool optimize_use_implicit_projections; + extern const SettingsBool optimize_use_projections; + extern const SettingsBool query_plan_aggregation_in_order; + extern const SettingsBool query_plan_convert_outer_join_to_inner_join; + extern const SettingsBool query_plan_enable_optimizations; + extern const SettingsBool query_plan_execute_functions_after_sorting; + extern const SettingsBool query_plan_filter_push_down; + extern const SettingsBool query_plan_lift_up_array_join; + extern const SettingsBool query_plan_lift_up_union; + extern const SettingsUInt64 query_plan_max_optimizations_to_apply; + extern const SettingsBool query_plan_merge_expressions; + extern const SettingsBool query_plan_merge_filters; + extern const SettingsBool query_plan_optimize_prewhere; + extern const SettingsBool query_plan_push_down_limit; + extern const SettingsBool query_plan_read_in_order; + extern const SettingsBool query_plan_remove_redundant_distinct; + extern const SettingsBool query_plan_remove_redundant_sorting; + extern const SettingsBool query_plan_reuse_storage_ordering_for_window_functions; + extern const SettingsBool query_plan_split_filter; +} QueryPlanOptimizationSettings QueryPlanOptimizationSettings::fromSettings(const Settings & from) { QueryPlanOptimizationSettings settings; - settings.optimize_plan = from.query_plan_enable_optimizations; - settings.max_optimizations_to_apply = from.query_plan_max_optimizations_to_apply; + settings.optimize_plan = from[Setting::query_plan_enable_optimizations]; + settings.max_optimizations_to_apply = from[Setting::query_plan_max_optimizations_to_apply]; - settings.lift_up_array_join = from.query_plan_enable_optimizations && from.query_plan_lift_up_array_join; + settings.lift_up_array_join = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_lift_up_array_join]; - settings.push_down_limit = from.query_plan_enable_optimizations && from.query_plan_push_down_limit; + settings.push_down_limit = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_push_down_limit]; - settings.split_filter = from.query_plan_enable_optimizations && from.query_plan_split_filter; + settings.split_filter = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_split_filter]; - settings.merge_expressions = from.query_plan_enable_optimizations && from.query_plan_merge_expressions; + settings.merge_expressions = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_merge_expressions]; - settings.merge_filters = from.query_plan_enable_optimizations && from.query_plan_merge_filters; + settings.merge_filters = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_merge_filters]; - settings.filter_push_down = from.query_plan_enable_optimizations && from.query_plan_filter_push_down; + settings.filter_push_down = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_filter_push_down]; - settings.convert_outer_join_to_inner_join = from.query_plan_enable_optimizations && from.query_plan_convert_outer_join_to_inner_join; + settings.convert_outer_join_to_inner_join = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_convert_outer_join_to_inner_join]; - settings.optimize_prewhere = from.query_plan_enable_optimizations && from.query_plan_optimize_prewhere; + settings.optimize_prewhere = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_optimize_prewhere]; - settings.execute_functions_after_sorting = from.query_plan_enable_optimizations && from.query_plan_execute_functions_after_sorting; + settings.execute_functions_after_sorting = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_execute_functions_after_sorting]; - settings.reuse_storage_ordering_for_window_functions = from.query_plan_enable_optimizations && from.query_plan_reuse_storage_ordering_for_window_functions; + settings.reuse_storage_ordering_for_window_functions = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_reuse_storage_ordering_for_window_functions]; - settings.lift_up_union = from.query_plan_enable_optimizations && from.query_plan_lift_up_union; + settings.lift_up_union = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_lift_up_union]; - settings.distinct_in_order = from.query_plan_enable_optimizations && from.optimize_distinct_in_order; + settings.distinct_in_order = from[Setting::query_plan_enable_optimizations] && from[Setting::optimize_distinct_in_order]; - settings.read_in_order = from.query_plan_enable_optimizations && from.optimize_read_in_order && from.query_plan_read_in_order; + settings.read_in_order = from[Setting::query_plan_enable_optimizations] && from[Setting::optimize_read_in_order] && from[Setting::query_plan_read_in_order]; - settings.aggregation_in_order = from.query_plan_enable_optimizations && from.optimize_aggregation_in_order && from.query_plan_aggregation_in_order; + settings.aggregation_in_order = from[Setting::query_plan_enable_optimizations] && from[Setting::optimize_aggregation_in_order] && from[Setting::query_plan_aggregation_in_order]; - settings.remove_redundant_sorting = from.query_plan_enable_optimizations && from.query_plan_remove_redundant_sorting; + settings.remove_redundant_sorting = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_remove_redundant_sorting]; - settings.aggregate_partitions_independently = from.query_plan_enable_optimizations && from.allow_aggregate_partitions_independently; + settings.aggregate_partitions_independently = from[Setting::query_plan_enable_optimizations] && from[Setting::allow_aggregate_partitions_independently]; - settings.remove_redundant_distinct = from.query_plan_enable_optimizations && from.query_plan_remove_redundant_distinct; + settings.remove_redundant_distinct = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_remove_redundant_distinct]; - settings.optimize_projection = from.optimize_use_projections; - settings.force_use_projection = settings.optimize_projection && from.force_optimize_projection; - settings.force_projection_name = settings.optimize_projection ? from.force_optimize_projection_name.value : ""; - settings.optimize_use_implicit_projections = settings.optimize_projection && from.optimize_use_implicit_projections; + settings.optimize_projection = from[Setting::optimize_use_projections]; + settings.force_use_projection = settings.optimize_projection && from[Setting::force_optimize_projection]; + settings.force_projection_name = settings.optimize_projection ? from[Setting::force_optimize_projection_name].value : ""; + settings.optimize_use_implicit_projections = settings.optimize_projection && from[Setting::optimize_use_implicit_projections]; return settings; } diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp index dc73521210a..39131955a4f 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_move_to_prewhere; + extern const SettingsBool optimize_move_to_prewhere_if_final; +} namespace QueryPlanOptimizations { @@ -69,8 +74,8 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &) const auto & settings = context->getSettingsRef(); bool is_final = source_step_with_filter->isQueryWithFinal(); - bool optimize_move_to_prewhere = settings.optimize_move_to_prewhere && (!is_final || settings.optimize_move_to_prewhere_if_final); - if (!optimize_move_to_prewhere) + bool optimize = settings[Setting::optimize_move_to_prewhere] && (!is_final || settings[Setting::optimize_move_to_prewhere_if_final]); + if (!optimize) return; const auto & storage_metadata = storage_snapshot->metadata; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 903d40509f5..e3fb211aeb2 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -30,6 +30,16 @@ #include +namespace DB +{ +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool query_plan_read_in_order; + extern const SettingsBool optimize_read_in_order; + extern const SettingsBool optimize_read_in_window_order; +} +} namespace DB::QueryPlanOptimizations { @@ -1081,7 +1091,8 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node, auto context = read_from_merge_tree->getContext(); const auto & settings = context->getSettingsRef(); - if (!settings.optimize_read_in_window_order || (settings.optimize_read_in_order && settings.query_plan_read_in_order) || context->getSettingsRef().allow_experimental_analyzer) + if (!settings[Setting::optimize_read_in_window_order] || (settings[Setting::optimize_read_in_order] && settings[Setting::query_plan_read_in_order]) + || context->getSettingsRef()[Setting::allow_experimental_analyzer]) { return 0; } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp index 35ac2c220ff..0b6362d3cd2 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp @@ -27,6 +27,14 @@ #include #include +namespace DB +{ +namespace Setting +{ + extern const SettingsString preferred_optimize_projection_name; +} +} + namespace DB::QueryPlanOptimizations { @@ -513,7 +521,7 @@ AggregateProjectionCandidates getAggregateProjectionCandidates( agg_projections.begin(), agg_projections.end(), [&](const auto * projection) - { return projection->name == context->getSettingsRef().preferred_optimize_projection_name.value; }); + { return projection->name == context->getSettingsRef()[Setting::preferred_optimize_projection_name].value; }); if (it != agg_projections.end()) { diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp index 654046cda89..aa266273e5b 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp @@ -14,6 +14,14 @@ #include +namespace DB +{ +namespace Setting +{ + extern const SettingsString preferred_optimize_projection_name; +} +} + namespace DB::QueryPlanOptimizations { @@ -115,7 +123,7 @@ std::optional optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod auto it = std::find_if( normal_projections.begin(), normal_projections.end(), - [&](const auto * projection) { return projection->name == context->getSettingsRef().preferred_optimize_projection_name.value; }); + [&](const auto * projection) { return projection->name == context->getSettingsRef()[Setting::preferred_optimize_projection_name].value; }); if (it != normal_projections.end()) { diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index 998b606ec57..5b0407352d1 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -15,6 +15,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool aggregate_functions_null_for_empty; + extern const SettingsBool allow_experimental_query_deduplication; + extern const SettingsBool apply_mutations_on_fly; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 select_sequential_consistency; +} namespace ErrorCodes { @@ -44,16 +52,16 @@ bool canUseProjectionForReadingStep(ReadFromMergeTree * reading) const auto & query_settings = reading->getContext()->getSettingsRef(); // Currently projection don't support deduplication when moving parts between shards. - if (query_settings.allow_experimental_query_deduplication) + if (query_settings[Setting::allow_experimental_query_deduplication]) return false; // Currently projection don't support settings which implicitly modify aggregate functions. - if (query_settings.aggregate_functions_null_for_empty) + if (query_settings[Setting::aggregate_functions_null_for_empty]) return false; /// Don't use projections if have mutations to apply /// because we need to apply them on original data. - if (query_settings.apply_mutations_on_fly && reading->getMutationsSnapshot()->hasDataMutations()) + if (query_settings[Setting::apply_mutations_on_fly] && reading->getMutationsSnapshot()->hasDataMutations()) return false; return true; @@ -63,7 +71,7 @@ std::shared_ptr getMaxAddedBlocks(ReadFromMergeTree * rea { ContextPtr context = reading->getContext(); - if (context->getSettingsRef().select_sequential_consistency) + if (context->getSettingsRef()[Setting::select_sequential_consistency]) { if (const auto * replicated = dynamic_cast(&reading->getMergeTreeData())) return std::make_shared(replicated->getMaxAddedBlocks()); @@ -248,7 +256,7 @@ bool analyzeProjectionCandidate( candidate.projection->metadata, projection_query_info, context, - context->getSettingsRef().max_threads, + context->getSettingsRef()[Setting::max_threads], max_added_blocks); candidate.merge_tree_projection_select_result_ptr = std::move(projection_result_ptr); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 60eadb80496..da88e59cd1c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -41,11 +41,8 @@ #include #include #include -#include -#include #include #include -#include #include #include @@ -132,6 +129,54 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_asynchronous_read_from_io_pool_for_merge_tree; + extern const SettingsBool allow_prefetched_read_pool_for_local_filesystem; + extern const SettingsBool allow_prefetched_read_pool_for_remote_filesystem; + extern const SettingsBool apply_deleted_mask; + extern const SettingsBool checksum_on_read; + extern const SettingsBool compile_sort_description; + extern const SettingsBool do_not_merge_across_partitions_select_final; + extern const SettingsBool enable_multiple_prewhere_read_steps; + extern const SettingsBool enable_vertical_final; + extern const SettingsBool force_aggregate_partitions_independently; + extern const SettingsBool force_primary_key; + extern const SettingsString ignore_data_skipping_indices; + extern const SettingsUInt64 max_number_of_partitions_for_independent_aggregation; + extern const SettingsUInt64 max_rows_to_read; + extern const SettingsUInt64 max_rows_to_read_leaf; + extern const SettingsMaxThreads max_final_threads; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsFloat max_streams_to_max_threads_ratio; + extern const SettingsUInt64 max_streams_for_merge_tree_reading; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 merge_tree_max_bytes_to_use_cache; + extern const SettingsUInt64 merge_tree_max_rows_to_use_cache; + extern const SettingsUInt64 merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem; + extern const SettingsUInt64 merge_tree_min_rows_for_concurrent_read_for_remote_filesystem; + extern const SettingsUInt64 merge_tree_min_bytes_for_concurrent_read; + extern const SettingsUInt64 merge_tree_min_rows_for_concurrent_read; + extern const SettingsFloat merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability; + extern const SettingsBool merge_tree_use_const_size_tasks_for_remote_reading; + extern const SettingsUInt64 min_count_to_compile_sort_description; + extern const SettingsOverflowMode read_overflow_mode; + extern const SettingsOverflowMode read_overflow_mode_leaf; + extern const SettingsUInt64 parallel_replicas_count; + extern const SettingsBool parallel_replicas_local_plan; + extern const SettingsFloat parallel_replicas_single_task_marks_count_multiplier; + extern const SettingsUInt64 preferred_block_size_bytes; + extern const SettingsUInt64 preferred_max_column_in_block_size_bytes; + extern const SettingsUInt64 read_in_order_two_level_merge_threshold; + extern const SettingsBool split_parts_ranges_into_intersecting_and_non_intersecting_final; + extern const SettingsBool split_intersecting_parts_ranges_into_layers_final; + extern const SettingsBool use_skip_indexes; + extern const SettingsBool use_skip_indexes_if_final; + extern const SettingsBool use_uncompressed_cache; +} + namespace ErrorCodes { extern const int INDEX_NOT_USED; @@ -145,16 +190,15 @@ static MergeTreeReaderSettings getMergeTreeReaderSettings( const ContextPtr & context, const SelectQueryInfo & query_info) { const auto & settings = context->getSettingsRef(); - return - { + return { .read_settings = context->getReadSettings(), .save_marks_in_cache = true, - .checksum_on_read = settings.checksum_on_read, + .checksum_on_read = settings[Setting::checksum_on_read], .read_in_order = query_info.input_order_info != nullptr, - .apply_deleted_mask = settings.apply_deleted_mask, - .use_asynchronous_read_from_pool = settings.allow_asynchronous_read_from_io_pool_for_merge_tree - && (settings.max_streams_to_max_threads_ratio > 1 || settings.max_streams_for_merge_tree_reading > 1), - .enable_multiple_prewhere_read_steps = settings.enable_multiple_prewhere_read_steps, + .apply_deleted_mask = settings[Setting::apply_deleted_mask], + .use_asynchronous_read_from_pool = settings[Setting::allow_asynchronous_read_from_io_pool_for_merge_tree] + && (settings[Setting::max_streams_to_max_threads_ratio] > 1 || settings[Setting::max_streams_for_merge_tree_reading] > 1), + .enable_multiple_prewhere_read_steps = settings[Setting::enable_multiple_prewhere_read_steps], }; } @@ -240,16 +284,12 @@ void ReadFromMergeTree::AnalysisResult::checkLimits(const Settings & settings, c /// because we can stop reading at any time. SizeLimits limits; - if (settings.read_overflow_mode == OverflowMode::THROW - && settings.max_rows_to_read - && !query_info_.input_order_info) - limits = SizeLimits(settings.max_rows_to_read, 0, settings.read_overflow_mode); + if (settings[Setting::read_overflow_mode] == OverflowMode::THROW && settings[Setting::max_rows_to_read] && !query_info_.input_order_info) + limits = SizeLimits(settings[Setting::max_rows_to_read], 0, settings[Setting::read_overflow_mode]); SizeLimits leaf_limits; - if (settings.read_overflow_mode_leaf == OverflowMode::THROW - && settings.max_rows_to_read_leaf - && !query_info_.input_order_info) - leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, 0, settings.read_overflow_mode_leaf); + if (settings[Setting::read_overflow_mode_leaf] == OverflowMode::THROW && settings[Setting::max_rows_to_read_leaf] && !query_info_.input_order_info) + leaf_limits = SizeLimits(settings[Setting::max_rows_to_read_leaf], 0, settings[Setting::read_overflow_mode_leaf]); if (limits.max_rows || leaf_limits.max_rows) { @@ -293,8 +333,8 @@ ReadFromMergeTree::ReadFromMergeTree( , actions_settings(ExpressionActionsSettings::fromContext(context_)) , block_size{ .max_block_size_rows = max_block_size_, - .preferred_block_size_bytes = context->getSettingsRef().preferred_block_size_bytes, - .preferred_max_column_in_block_size_bytes = context->getSettingsRef().preferred_max_column_in_block_size_bytes} + .preferred_block_size_bytes = context->getSettingsRef()[Setting::preferred_block_size_bytes], + .preferred_max_column_in_block_size_bytes = context->getSettingsRef()[Setting::preferred_max_column_in_block_size_bytes]} , requested_num_streams(num_streams_) , max_block_numbers_to_read(std::move(max_block_numbers_to_read_)) , log(std::move(log_)) @@ -317,26 +357,27 @@ ReadFromMergeTree::ReadFromMergeTree( } const auto & settings = context->getSettingsRef(); - if (settings.max_streams_for_merge_tree_reading) + if (settings[Setting::max_streams_for_merge_tree_reading]) { - if (settings.allow_asynchronous_read_from_io_pool_for_merge_tree) + if (settings[Setting::allow_asynchronous_read_from_io_pool_for_merge_tree]) { /// When async reading is enabled, allow to read using more streams. /// Will add resize to output_streams_limit to reduce memory usage. - output_streams_limit = std::min(requested_num_streams, settings.max_streams_for_merge_tree_reading); + output_streams_limit = std::min(requested_num_streams, settings[Setting::max_streams_for_merge_tree_reading]); /// We intentionally set `max_streams` to 1 in InterpreterSelectQuery in case of small limit. /// Changing it here to `max_streams_for_merge_tree_reading` proven itself as a threat for performance. if (requested_num_streams != 1) - requested_num_streams = std::max(requested_num_streams, settings.max_streams_for_merge_tree_reading); + requested_num_streams = std::max(requested_num_streams, settings[Setting::max_streams_for_merge_tree_reading]); } else /// Just limit requested_num_streams otherwise. - requested_num_streams = std::min(requested_num_streams, settings.max_streams_for_merge_tree_reading); + requested_num_streams = std::min(requested_num_streams, settings[Setting::max_streams_for_merge_tree_reading]); } /// Add explicit description. setStepDescription(data.getStorageID().getFullNameNotQuoted()); - enable_vertical_final = query_info.isFinal() && context->getSettingsRef().enable_vertical_final && data.merging_params.mode == MergeTreeData::MergingParams::Replacing; + enable_vertical_final = query_info.isFinal() && context->getSettingsRef()[Setting::enable_vertical_final] + && data.merging_params.mode == MergeTreeData::MergingParams::Replacing; updateSortDescriptionForOutputStream( *output_stream, @@ -387,11 +428,11 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit /// We have a special logic for local replica. It has to read less data, because in some cases it should /// merge states of aggregate functions or do some other important stuff other than reading from Disk. - auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; + auto multiplier = context->getSettingsRef()[Setting::parallel_replicas_single_task_marks_count_multiplier]; const auto min_marks_for_concurrent_read_limit = std::numeric_limits::max() >> 1; if (pool_settings.min_marks_for_concurrent_read > min_marks_for_concurrent_read_limit) { - /// limit min marks to read in case it's big, happened in test since due to settings randomzation + /// limit min marks to read in case it's big, happened in test since due to settings randomization pool_settings.min_marks_for_concurrent_read = min_marks_for_concurrent_read_limit; multiplier = 1.0f; } @@ -471,12 +512,10 @@ Pipe ReadFromMergeTree::readFromPool( MergeTreeReadPoolPtr pool; - bool allow_prefetched_remote = all_parts_are_remote - && settings.allow_prefetched_read_pool_for_remote_filesystem + bool allow_prefetched_remote = all_parts_are_remote && settings[Setting::allow_prefetched_read_pool_for_remote_filesystem] && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method); - bool allow_prefetched_local = all_parts_are_local - && settings.allow_prefetched_read_pool_for_local_filesystem + bool allow_prefetched_local = all_parts_are_local && settings[Setting::allow_prefetched_read_pool_for_local_filesystem] && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method); /** Do not use prefetched read pool if query is trivial limit query. @@ -568,7 +607,7 @@ Pipe ReadFromMergeTree::readInOrder( .total_nodes_count = context->getClusterForParallelReplicas()->getShardsInfo().at(0).getAllNodeCount(), }; - auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; + auto multiplier = context->getSettingsRef()[Setting::parallel_replicas_single_task_marks_count_multiplier]; const auto min_marks_for_concurrent_read_limit = std::numeric_limits::max() >> 1; if (pool_settings.min_marks_for_concurrent_read > min_marks_for_concurrent_read_limit) { @@ -624,7 +663,7 @@ Pipe ReadFromMergeTree::readInOrder( /// Otherwise rows will counted multiple times const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; const bool parallel_replicas_local_plan_for_initiator = is_parallel_reading_from_replicas - && context->getSettingsRef().parallel_replicas_local_plan && context->canUseParallelReplicasOnInitiator(); + && context->getSettingsRef()[Setting::parallel_replicas_local_plan] && context->canUseParallelReplicasOnInitiator(); const bool set_total_rows_approx = !is_parallel_reading_from_replicas || parallel_replicas_local_plan_for_initiator; Pipes pipes; @@ -687,14 +726,13 @@ Pipe ReadFromMergeTree::read( const auto & settings = context->getSettingsRef(); size_t sum_marks = parts_with_range.getMarksCountAllParts(); - PoolSettings pool_settings - { + PoolSettings pool_settings{ .threads = max_streams, .sum_marks = sum_marks, .min_marks_for_concurrent_read = min_marks_for_concurrent_read, - .preferred_block_size_bytes = settings.preferred_block_size_bytes, + .preferred_block_size_bytes = settings[Setting::preferred_block_size_bytes], .use_uncompressed_cache = use_uncompressed_cache, - .use_const_size_tasks_for_remote_reading = settings.merge_tree_use_const_size_tasks_for_remote_reading, + .use_const_size_tasks_for_remote_reading = settings[Setting::merge_tree_use_const_size_tasks_for_remote_reading], }; if (read_type == ReadType::ParallelReplicas) @@ -751,8 +789,8 @@ struct PartRangesReadInfo index_granularity_bytes = data_settings.index_granularity_bytes; max_marks_to_use_cache = MergeTreeDataSelectExecutor::roundRowsOrBytesToMarks( - settings.merge_tree_max_rows_to_use_cache, - settings.merge_tree_max_bytes_to_use_cache, + settings[Setting::merge_tree_max_rows_to_use_cache], + settings[Setting::merge_tree_max_bytes_to_use_cache], data_settings.index_granularity, index_granularity_bytes); @@ -762,20 +800,20 @@ struct PartRangesReadInfo size_t min_bytes_for_concurrent_read; if (all_parts_on_remote_disk) { - min_rows_for_concurrent_read = settings.merge_tree_min_rows_for_concurrent_read_for_remote_filesystem; - min_bytes_for_concurrent_read = settings.merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem; + min_rows_for_concurrent_read = settings[Setting::merge_tree_min_rows_for_concurrent_read_for_remote_filesystem]; + min_bytes_for_concurrent_read = settings[Setting::merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem]; } else { - min_rows_for_concurrent_read = settings.merge_tree_min_rows_for_concurrent_read; - min_bytes_for_concurrent_read = settings.merge_tree_min_bytes_for_concurrent_read; + min_rows_for_concurrent_read = settings[Setting::merge_tree_min_rows_for_concurrent_read]; + min_bytes_for_concurrent_read = settings[Setting::merge_tree_min_bytes_for_concurrent_read]; } min_marks_for_concurrent_read = MergeTreeDataSelectExecutor::minMarksForConcurrentRead( min_rows_for_concurrent_read, min_bytes_for_concurrent_read, data_settings.index_granularity, index_granularity_bytes, sum_marks); - use_uncompressed_cache = settings.use_uncompressed_cache; + use_uncompressed_cache = settings[Setting::use_uncompressed_cache]; if (sum_marks > max_marks_to_use_cache) use_uncompressed_cache = false; } @@ -826,7 +864,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_ auto read_type = is_parallel_reading_from_replicas ? ReadType::ParallelReplicas : ReadType::Default; - double read_split_ranges_into_intersecting_and_non_intersecting_injection_probability = settings.merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability; + double read_split_ranges_into_intersecting_and_non_intersecting_injection_probability + = settings[Setting::merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability]; std::bernoulli_distribution fault(read_split_ranges_into_intersecting_and_non_intersecting_injection_probability); if (read_type != ReadType::ParallelReplicas && @@ -996,16 +1035,15 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( }; const size_t min_marks_per_stream = (info.sum_marks - 1) / num_streams + 1; - bool need_preliminary_merge = (parts_with_ranges.size() > settings.read_in_order_two_level_merge_threshold); + bool need_preliminary_merge = (parts_with_ranges.size() > settings[Setting::read_in_order_two_level_merge_threshold]); const auto read_type = input_order_info->direction == 1 ? ReadType::InOrder : ReadType::InReverseOrder; - PoolSettings pool_settings - { + PoolSettings pool_settings{ .threads = num_streams, .sum_marks = parts_with_ranges.getMarksCountAllParts(), .min_marks_for_concurrent_read = info.min_marks_for_concurrent_read, - .preferred_block_size_bytes = settings.preferred_block_size_bytes, + .preferred_block_size_bytes = settings[Setting::preferred_block_size_bytes], .use_uncompressed_cache = info.use_uncompressed_cache, }; @@ -1105,8 +1143,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( const auto & sorting_columns = storage_snapshot->metadata->getSortingKey().column_names; SortDescription sort_description; - sort_description.compile_sort_description = settings.compile_sort_description; - sort_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description; + sort_description.compile_sort_description = settings[Setting::compile_sort_description]; + sort_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description]; for (size_t j = 0; j < prefix_size; ++j) sort_description.emplace_back(sorting_columns[j], input_order_info->direction); @@ -1185,6 +1223,7 @@ static void addMergingFinal( return std::make_shared(header, num_outputs, sort_description, merging_params.is_deleted_column, merging_params.version_column, max_block_size_rows, /*max_block_size_bytes=*/0, /*out_row_sources_buf_*/ nullptr, /*use_average_block_sizes*/ false, /*cleanup*/ !merging_params.is_deleted_column.empty(), enable_vertical_final); + case MergeTreeData::MergingParams::VersionedCollapsing: return std::make_shared(header, num_outputs, sort_description, merging_params.sign_column, max_block_size_rows, /*max_block_size_bytes=*/0); @@ -1206,8 +1245,8 @@ bool ReadFromMergeTree::doNotMergePartsAcrossPartitionsFinal() const const auto & settings = context->getSettingsRef(); /// If setting do_not_merge_across_partitions_select_final is set always prefer it - if (settings.do_not_merge_across_partitions_select_final.changed) - return settings.do_not_merge_across_partitions_select_final; + if (settings[Setting::do_not_merge_across_partitions_select_final].changed) + return settings[Setting::do_not_merge_across_partitions_select_final]; if (!storage_snapshot->metadata->hasPrimaryKey() || !storage_snapshot->metadata->hasPartitionKey()) return false; @@ -1242,7 +1281,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( PartRangesReadInfo info(parts_with_ranges, settings, *data_settings); assert(num_streams == requested_num_streams); - num_streams = std::min(num_streams, settings.max_final_threads); + num_streams = std::min(num_streams, settings[Setting::max_final_threads]); /// If setting do_not_merge_across_partitions_select_final is true than we won't merge parts from different partitions. /// We have all parts in parts vector, where parts with same partition are nearby. @@ -1328,8 +1367,9 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( /// Parts of non-zero level still may contain duplicate PK values to merge on FINAL if there's is_deleted column, /// so we have to process all ranges. It would be more optimal to remove this flag and add an extra filtering step. - bool split_parts_ranges_into_intersecting_and_non_intersecting_final = settings.split_parts_ranges_into_intersecting_and_non_intersecting_final && - data.merging_params.is_deleted_column.empty() && !reader_settings.read_in_order; + bool split_parts_ranges_into_intersecting_and_non_intersecting_final + = settings[Setting::split_parts_ranges_into_intersecting_and_non_intersecting_final] + && data.merging_params.is_deleted_column.empty() && !reader_settings.read_in_order; SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey( storage_snapshot->metadata->getPrimaryKey(), @@ -1339,7 +1379,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( context, std::move(in_order_reading_step_getter), split_parts_ranges_into_intersecting_and_non_intersecting_final, - settings.split_intersecting_parts_ranges_into_layers_final); + settings[Setting::split_intersecting_parts_ranges_into_layers_final]); for (auto && non_intersecting_parts_range : split_ranges_result.non_intersecting_parts_ranges) non_intersecting_parts_by_primary_key.push_back(std::move(non_intersecting_parts_range)); @@ -1366,8 +1406,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns(); SortDescription sort_description; - sort_description.compile_sort_description = settings.compile_sort_description; - sort_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description; + sort_description.compile_sort_description = settings[Setting::compile_sort_description]; + sort_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description]; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); @@ -1475,8 +1515,8 @@ static void buildIndexes( indexes->part_values = MergeTreeDataSelectExecutor::filterPartsByVirtualColumns(metadata_snapshot, data, parts, filter_actions_dag, context); MergeTreeDataSelectExecutor::buildKeyConditionFromPartOffset(indexes->part_offset_condition, filter_actions_dag, context); - indexes->use_skip_indexes = settings.use_skip_indexes; - if (query_info.isFinal() && !settings.use_skip_indexes_if_final) + indexes->use_skip_indexes = settings[Setting::use_skip_indexes]; + if (query_info.isFinal() && !settings[Setting::use_skip_indexes_if_final]) indexes->use_skip_indexes = false; if (!indexes->use_skip_indexes) @@ -1489,11 +1529,11 @@ static void buildIndexes( std::unordered_set ignored_index_names; - if (settings.ignore_data_skipping_indices.changed) + if (settings[Setting::ignore_data_skipping_indices].changed) { - const auto & indices = settings.ignore_data_skipping_indices.toString(); - Tokens tokens(indices.data(), indices.data() + indices.size(), settings.max_query_size); - IParser::Pos pos(tokens, static_cast(settings.max_parser_depth), static_cast(settings.max_parser_backtracks)); + const auto & indices = settings[Setting::ignore_data_skipping_indices].toString(); + Tokens tokens(indices.data(), indices.data() + indices.size(), settings[Setting::max_query_size]); + IParser::Pos pos(tokens, static_cast(settings[Setting::max_parser_depth]), static_cast(settings[Setting::max_parser_backtracks])); Expected expected; /// Use an unordered list rather than string vector @@ -1658,7 +1698,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( if (indexes->key_condition.alwaysUnknownOrTrue()) { - if (settings.force_primary_key) + if (settings[Setting::force_primary_key]) { throw Exception(ErrorCodes::INDEX_NOT_USED, "Primary key ({}) is not used and setting 'force_primary_key' is set", @@ -1843,7 +1883,7 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() const auto & settings = context->getSettingsRef(); const auto partitions_cnt = countPartitions(prepared_parts); - if (!settings.force_aggregate_partitions_independently && (partitions_cnt == 1 || partitions_cnt < settings.max_threads / 2)) + if (!settings[Setting::force_aggregate_partitions_independently] && (partitions_cnt == 1 || partitions_cnt < settings[Setting::max_threads] / 2)) { LOG_TRACE( log, @@ -1853,8 +1893,8 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() return false; } - if (!settings.force_aggregate_partitions_independently - && (partitions_cnt > settings.max_number_of_partitions_for_independent_aggregation)) + if (!settings[Setting::force_aggregate_partitions_independently] + && (partitions_cnt > settings[Setting::max_number_of_partitions_for_independent_aggregation])) { LOG_TRACE( log, @@ -1862,11 +1902,11 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() "max_number_of_partitions_for_independent_aggregation (current value is {}) or set " "force_aggregate_partitions_independently to suppress this check", partitions_cnt, - settings.max_number_of_partitions_for_independent_aggregation); + settings[Setting::max_number_of_partitions_for_independent_aggregation]); return false; } - if (!settings.force_aggregate_partitions_independently) + if (!settings[Setting::force_aggregate_partitions_independently]) { std::unordered_map partition_rows; for (const auto & part : prepared_parts) @@ -1882,7 +1922,7 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort() /// Merging shouldn't take more time than preaggregation in normal cases. And exec time is proportional to the amount of data. /// We assume that exec time of independent aggr is proportional to the maximum of sizes and /// exec time of ordinary aggr is proportional to sum of sizes divided by number of threads and multiplied by two (preaggregation + merging). - const size_t avg_rows_in_partition = sum_rows / settings.max_threads; + const size_t avg_rows_in_partition = sum_rows / settings[Setting::max_threads]; if (max_rows > avg_rows_in_partition * 2) { LOG_TRACE( @@ -1906,7 +1946,7 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const bool ReadFromMergeTree::isQueryWithSampling() const { - if (context->getSettingsRef().parallel_replicas_count > 1 && data.supportsSampling()) + if (context->getSettingsRef()[Setting::parallel_replicas_count] > 1 && data.supportsSampling()) return true; const auto & select = query_info.query->as(); diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 3df46eb1987..7a216ab5716 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -26,6 +26,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool async_query_sending_for_remote; + extern const SettingsBool async_socket_for_remote; + extern const SettingsString cluster_for_parallel_replicas; + extern const SettingsBool extremes; + extern const SettingsSeconds max_execution_time; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsUInt64 parallel_replicas_mark_segment_size; +} namespace ErrorCodes { @@ -144,13 +154,13 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream bool add_agg_info = stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; bool add_extremes = false; - bool async_read = context->getSettingsRef().async_socket_for_remote; - const bool async_query_sending = context->getSettingsRef().async_query_sending_for_remote; + bool async_read = context->getSettingsRef()[Setting::async_socket_for_remote]; + const bool async_query_sending = context->getSettingsRef()[Setting::async_query_sending_for_remote]; if (stage == QueryProcessingStage::Complete) { add_totals = shard.query->as().group_by_with_totals; - add_extremes = context->getSettingsRef().extremes; + add_extremes = context->getSettingsRef()[Setting::extremes]; } auto lazily_create_stream = [ @@ -164,7 +174,7 @@ void ReadFromRemote::addLazyPipe(Pipes & pipes, const ClusterProxy::SelectStream { auto current_settings = my_context->getSettingsRef(); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings) - .getSaturated(current_settings.max_execution_time); + .getSaturated(current_settings[Setting::max_execution_time]); std::vector try_results; try @@ -231,12 +241,12 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact bool add_agg_info = stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; bool add_extremes = false; - bool async_read = context->getSettingsRef().async_socket_for_remote; - bool async_query_sending = context->getSettingsRef().async_query_sending_for_remote; + bool async_read = context->getSettingsRef()[Setting::async_socket_for_remote]; + bool async_query_sending = context->getSettingsRef()[Setting::async_query_sending_for_remote]; if (stage == QueryProcessingStage::Complete) { add_totals = shard.query->as().group_by_with_totals; - add_extremes = context->getSettingsRef().extremes; + add_extremes = context->getSettingsRef()[Setting::extremes]; } scalars["_shard_num"] @@ -244,9 +254,9 @@ void ReadFromRemote::addPipe(Pipes & pipes, const ClusterProxy::SelectStreamFact if (context->canUseTaskBasedParallelReplicas()) { - if (context->getSettingsRef().cluster_for_parallel_replicas.changed) + if (context->getSettingsRef()[Setting::cluster_for_parallel_replicas].changed) { - const String cluster_for_parallel_replicas = context->getSettingsRef().cluster_for_parallel_replicas; + const String cluster_for_parallel_replicas = context->getSettingsRef()[Setting::cluster_for_parallel_replicas]; if (cluster_for_parallel_replicas != cluster_name) LOG_INFO( log, @@ -458,13 +468,13 @@ void ReadFromParallelRemoteReplicasStep::addPipeForSingeReplica( bool add_agg_info = stage == QueryProcessingStage::WithMergeableState; bool add_totals = false; bool add_extremes = false; - bool async_read = context->getSettingsRef().async_socket_for_remote; - bool async_query_sending = context->getSettingsRef().async_query_sending_for_remote; + bool async_read = context->getSettingsRef()[Setting::async_socket_for_remote]; + bool async_query_sending = context->getSettingsRef()[Setting::async_query_sending_for_remote]; if (stage == QueryProcessingStage::Complete) { add_totals = query_ast->as().group_by_with_totals; - add_extremes = context->getSettingsRef().extremes; + add_extremes = context->getSettingsRef()[Setting::extremes]; } String query_string = formattedAST(query_ast); diff --git a/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp b/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp index 51117ad3c9b..66db1e77bdd 100644 --- a/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp +++ b/src/Processors/QueryPlan/ReadFromStreamLikeEngine.cpp @@ -6,6 +6,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool stream_like_engine_allow_direct_select; +} namespace ErrorCodes { @@ -25,7 +29,7 @@ ReadFromStreamLikeEngine::ReadFromStreamLikeEngine( void ReadFromStreamLikeEngine::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - if (!getContext()->getSettingsRef().stream_like_engine_allow_direct_select) + if (!getContext()->getSettingsRef()[Setting::stream_like_engine_allow_direct_select]) throw Exception( ErrorCodes::QUERY_NOT_ALLOWED, "Direct select is not allowed. To enable use setting `stream_like_engine_allow_direct_select`"); diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 596d08845e1..2855793ea2c 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -18,6 +18,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_rows_to_read; + extern const SettingsUInt64 max_rows_to_read_leaf; + extern const SettingsOverflowMode read_overflow_mode; + extern const SettingsOverflowMode read_overflow_mode_leaf; +} namespace ErrorCodes { @@ -614,15 +621,15 @@ void ReadFromSystemNumbersStep::checkLimits(size_t rows) { const auto & settings = context->getSettingsRef(); - if (settings.read_overflow_mode == OverflowMode::THROW && settings.max_rows_to_read) + if (settings[Setting::read_overflow_mode] == OverflowMode::THROW && settings[Setting::max_rows_to_read]) { - const auto limits = SizeLimits(settings.max_rows_to_read, 0, settings.read_overflow_mode); + const auto limits = SizeLimits(settings[Setting::max_rows_to_read], 0, settings[Setting::read_overflow_mode]); limits.check(rows, 0, "rows (controlled by 'max_rows_to_read' setting)", ErrorCodes::TOO_MANY_ROWS); } - if (settings.read_overflow_mode_leaf == OverflowMode::THROW && settings.max_rows_to_read_leaf) + if (settings[Setting::read_overflow_mode_leaf] == OverflowMode::THROW && settings[Setting::max_rows_to_read_leaf]) { - const auto leaf_limits = SizeLimits(settings.max_rows_to_read_leaf, 0, settings.read_overflow_mode_leaf); + const auto leaf_limits = SizeLimits(settings[Setting::max_rows_to_read_leaf], 0, settings[Setting::read_overflow_mode_leaf]); leaf_limits.check(rows, 0, "rows (controlled by 'max_rows_to_read_leaf' setting)", ErrorCodes::TOO_MANY_ROWS); } } diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 48fad9f5fdb..58ccdf127bc 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -23,6 +23,19 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_bytes_before_external_sort; + extern const SettingsUInt64 max_bytes_before_remerge_sort; + extern const SettingsUInt64 max_bytes_to_sort; + extern const SettingsUInt64 max_rows_to_sort; + extern const SettingsUInt64 min_free_disk_space_for_temporary_data; + extern const SettingsUInt64 prefer_external_sort_block_bytes; + extern const SettingsBool read_in_order_use_buffering; + extern const SettingsFloat remerge_sort_lowered_memory_bytes_ratio; + extern const SettingsOverflowMode sort_overflow_mode; +} namespace ErrorCodes { @@ -32,15 +45,15 @@ namespace ErrorCodes SortingStep::Settings::Settings(const Context & context) { const auto & settings = context.getSettingsRef(); - max_block_size = settings.max_block_size; - size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode); - max_bytes_before_remerge = settings.max_bytes_before_remerge_sort; - remerge_lowered_memory_bytes_ratio = settings.remerge_sort_lowered_memory_bytes_ratio; - max_bytes_before_external_sort = settings.max_bytes_before_external_sort; + max_block_size = settings[Setting::max_block_size]; + size_limits = SizeLimits(settings[Setting::max_rows_to_sort], settings[Setting::max_bytes_to_sort], settings[Setting::sort_overflow_mode]); + max_bytes_before_remerge = settings[Setting::max_bytes_before_remerge_sort]; + remerge_lowered_memory_bytes_ratio = settings[Setting::remerge_sort_lowered_memory_bytes_ratio]; + max_bytes_before_external_sort = settings[Setting::max_bytes_before_external_sort]; tmp_data = context.getTempDataOnDisk(); - min_free_disk_space = settings.min_free_disk_space_for_temporary_data; - max_block_bytes = settings.prefer_external_sort_block_bytes; - read_in_order_use_buffering = settings.read_in_order_use_buffering; + min_free_disk_space = settings[Setting::min_free_disk_space_for_temporary_data]; + max_block_bytes = settings[Setting::prefer_external_sort_block_bytes]; + read_in_order_use_buffering = settings[Setting::read_in_order_use_buffering]; } SortingStep::Settings::Settings(size_t max_block_size_) diff --git a/src/Processors/Sources/MySQLSource.cpp b/src/Processors/Sources/MySQLSource.cpp index 52be9a6e84a..56f2af93dbb 100644 --- a/src/Processors/Sources/MySQLSource.cpp +++ b/src/Processors/Sources/MySQLSource.cpp @@ -28,6 +28,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 external_storage_max_read_bytes; + extern const SettingsUInt64 external_storage_max_read_rows; + extern const SettingsUInt64 max_block_size; +} namespace ErrorCodes { @@ -36,8 +42,9 @@ namespace ErrorCodes } StreamSettings::StreamSettings(const Settings & settings, bool auto_close_, bool fetch_by_name_, size_t max_retry_) - : max_read_mysql_row_nums((settings.external_storage_max_read_rows) ? settings.external_storage_max_read_rows : settings.max_block_size) - , max_read_mysql_bytes_size(settings.external_storage_max_read_bytes) + : max_read_mysql_row_nums( + (settings[Setting::external_storage_max_read_rows]) ? settings[Setting::external_storage_max_read_rows] : settings[Setting::max_block_size]) + , max_read_mysql_bytes_size(settings[Setting::external_storage_max_read_bytes]) , auto_close(auto_close_) , fetch_by_name(fetch_by_name_) , default_num_tries_on_connection_loss(max_retry_) diff --git a/src/Processors/Sources/RecursiveCTESource.cpp b/src/Processors/Sources/RecursiveCTESource.cpp index 40e72e9cbb1..7ddee23ded8 100644 --- a/src/Processors/Sources/RecursiveCTESource.cpp +++ b/src/Processors/Sources/RecursiveCTESource.cpp @@ -21,6 +21,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_recursive_cte_evaluation_depth; +} namespace ErrorCodes { @@ -162,12 +166,12 @@ private: { const auto & recursive_subquery_settings = recursive_query_context->getSettingsRef(); - if (recursive_step > recursive_subquery_settings.max_recursive_cte_evaluation_depth) + if (recursive_step > recursive_subquery_settings[Setting::max_recursive_cte_evaluation_depth]) throw Exception( ErrorCodes::TOO_DEEP_RECURSION, "Maximum recursive CTE evaluation depth ({}) exceeded, during evaluation of {}. Consider raising " "max_recursive_cte_evaluation_depth setting.", - recursive_subquery_settings.max_recursive_cte_evaluation_depth, + recursive_subquery_settings[Setting::max_recursive_cte_evaluation_depth], recursive_cte_union_node->formatASTForErrorMessage()); auto & query_to_execute = recursive_step > 0 ? recursive_query : non_recursive_query; diff --git a/src/Processors/Sources/WaitForAsyncInsertSource.h b/src/Processors/Sources/WaitForAsyncInsertSource.h index 78af6294202..c311a2e342f 100644 --- a/src/Processors/Sources/WaitForAsyncInsertSource.h +++ b/src/Processors/Sources/WaitForAsyncInsertSource.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB { diff --git a/src/Processors/TTL/TTLAggregationAlgorithm.cpp b/src/Processors/TTL/TTLAggregationAlgorithm.cpp index 2d7a37d0abe..225ff1582c0 100644 --- a/src/Processors/TTL/TTLAggregationAlgorithm.cpp +++ b/src/Processors/TTL/TTLAggregationAlgorithm.cpp @@ -4,6 +4,21 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool compile_aggregate_expressions; + extern const SettingsBool empty_result_for_aggregation_by_empty_set; + extern const SettingsBool enable_software_prefetch_in_aggregation; + extern const SettingsOverflowModeGroupBy group_by_overflow_mode; + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_bytes_before_external_group_by; + extern const SettingsUInt64 max_rows_to_group_by; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_chunk_bytes_for_parallel_parsing; + extern const SettingsUInt64 min_count_to_compile_aggregate_expression; + extern const SettingsUInt64 min_free_disk_space_for_temporary_data; + extern const SettingsBool optimize_group_by_constant_keys; +} TTLAggregationAlgorithm::TTLAggregationAlgorithm( const TTLExpressions & ttl_expressions_, @@ -30,23 +45,23 @@ TTLAggregationAlgorithm::TTLAggregationAlgorithm( keys, aggregates, false, - settings.max_rows_to_group_by, - settings.group_by_overflow_mode, - /*group_by_two_level_threshold*/0, - /*group_by_two_level_threshold_bytes*/0, - settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set, + settings[Setting::max_rows_to_group_by], + settings[Setting::group_by_overflow_mode], + /*group_by_two_level_threshold*/ 0, + /*group_by_two_level_threshold_bytes*/ 0, + settings[Setting::max_bytes_before_external_group_by], + settings[Setting::empty_result_for_aggregation_by_empty_set], storage_.getContext()->getTempDataOnDisk(), - settings.max_threads, - settings.min_free_disk_space_for_temporary_data, - settings.compile_aggregate_expressions, - settings.min_count_to_compile_aggregate_expression, - settings.max_block_size, - settings.enable_software_prefetch_in_aggregation, - /*only_merge=*/ false, - settings.optimize_group_by_constant_keys, - settings.min_chunk_bytes_for_parallel_parsing, - /*stats_collecting_params=*/ {}); + settings[Setting::max_threads], + settings[Setting::min_free_disk_space_for_temporary_data], + settings[Setting::compile_aggregate_expressions], + settings[Setting::min_count_to_compile_aggregate_expression], + settings[Setting::max_block_size], + settings[Setting::enable_software_prefetch_in_aggregation], + /*only_merge=*/false, + settings[Setting::optimize_group_by_constant_keys], + settings[Setting::min_chunk_bytes_for_parallel_parsing], + /*stats_collecting_params=*/{}); aggregator = std::make_unique(header, params); diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 98d66ed77c3..9d914149b4a 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -44,6 +44,26 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool deduplicate_blocks_in_dependent_materialized_views; + extern const SettingsBool ignore_materialized_views_with_dropped_target_table; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool log_queries; + extern const SettingsMilliseconds log_queries_min_query_duration_ms; + extern const SettingsLogQueriesType log_queries_min_type; + extern const SettingsBool log_query_views; + extern const SettingsBool materialized_views_ignore_errors; + extern const SettingsUInt64 max_block_size; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_bytes_for_materialized_views; + extern const SettingsUInt64 min_insert_block_size_rows; + extern const SettingsUInt64 min_insert_block_size_rows_for_materialized_views; + extern const SettingsBool parallel_view_processing; + extern const SettingsBool use_concurrency_control; +} namespace ErrorCodes { @@ -234,10 +254,11 @@ std::optional generateViewChain( select_context->setSetting("parallelize_output_from_storages", Field{false}); // Separate min_insert_block_size_rows/min_insert_block_size_bytes for children - if (insert_settings.min_insert_block_size_rows_for_materialized_views) - insert_context->setSetting("min_insert_block_size_rows", insert_settings.min_insert_block_size_rows_for_materialized_views.value); - if (insert_settings.min_insert_block_size_bytes_for_materialized_views) - insert_context->setSetting("min_insert_block_size_bytes", insert_settings.min_insert_block_size_bytes_for_materialized_views.value); + if (insert_settings[Setting::min_insert_block_size_rows_for_materialized_views]) + insert_context->setSetting("min_insert_block_size_rows", insert_settings[Setting::min_insert_block_size_rows_for_materialized_views].value); + if (insert_settings[Setting::min_insert_block_size_bytes_for_materialized_views]) + insert_context->setSetting( + "min_insert_block_size_bytes", insert_settings[Setting::min_insert_block_size_bytes_for_materialized_views].value); ASTPtr query; Chain out; @@ -269,7 +290,7 @@ std::optional generateViewChain( if (auto * materialized_view = dynamic_cast(view.get())) { - auto lock = materialized_view->tryLockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout); + auto lock = materialized_view->tryLockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); if (lock == nullptr) { @@ -287,7 +308,7 @@ std::optional generateViewChain( /// If target table was dropped, ignore this materialized view. if (!inner_table) { - if (context->getSettingsRef().ignore_materialized_views_with_dropped_target_table) + if (context->getSettingsRef()[Setting::ignore_materialized_views_with_dropped_target_table]) return std::nullopt; throw Exception( @@ -319,7 +340,7 @@ std::optional generateViewChain( Block header; /// Get list of columns we get from select query. - if (select_context->getSettingsRef().allow_experimental_analyzer) + if (select_context->getSettingsRef()[Setting::allow_experimental_analyzer]) header = InterpreterSelectQueryAnalyzer::getSampleBlock(query, select_context); else header = InterpreterSelectQuery(query, select_context, SelectQueryOptions()).getSampleBlock(); @@ -353,8 +374,8 @@ std::optional generateViewChain( out.addSource(std::make_shared( out.getInputHeader(), - table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size, - table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL)); + table_prefers_large_blocks ? settings[Setting::min_insert_block_size_rows] : settings[Setting::max_block_size], + table_prefers_large_blocks ? settings[Setting::min_insert_block_size_bytes] : 0ULL)); } #ifdef ABORT_ON_LOGICAL_ERROR @@ -454,9 +475,9 @@ Chain buildPushingToViewsChain( * Although now any insertion into the table is done via PushingToViews chain, * but it's clear that here is not the best place for this functionality. */ - result_chain.addTableLock(storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout)); + result_chain.addTableLock(storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout])); - bool disable_deduplication_for_children = !context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views; + bool disable_deduplication_for_children = !context->getSettingsRef()[Setting::deduplicate_blocks_in_dependent_materialized_views]; auto table_id = storage->getStorageID(); auto views = DatabaseCatalog::instance().getDependentViews(table_id); @@ -498,7 +519,7 @@ Chain buildPushingToViewsChain( catch (const Exception & e) { LOG_ERROR(&Poco::Logger::get("PushingToViews"), "Failed to push block to view {}, {}", view_id, e.message()); - if (!context->getSettingsRef().materialized_views_ignore_errors) + if (!context->getSettingsRef()[Setting::materialized_views_ignore_errors]) throw; } } @@ -507,8 +528,8 @@ Chain buildPushingToViewsChain( { size_t num_views = views_data->views.size(); const Settings & settings = context->getSettingsRef(); - if (settings.parallel_view_processing) - views_data->max_threads = settings.max_threads ? std::min(static_cast(settings.max_threads), num_views) : num_views; + if (settings[Setting::parallel_view_processing]) + views_data->max_threads = settings[Setting::max_threads] ? std::min(static_cast(settings[Setting::max_threads]), num_views) : num_views; std::vector headers; headers.reserve(num_views); @@ -539,7 +560,7 @@ Chain buildPushingToViewsChain( processors.emplace_back(std::move(finalizing_views)); result_chain = Chain(std::move(processors)); result_chain.setNumThreads(std::min(views_data->max_threads, max_parallel_streams)); - result_chain.setConcurrencyControl(settings.use_concurrency_control); + result_chain.setConcurrencyControl(settings[Setting::use_concurrency_control]); } if (auto * live_view = dynamic_cast(storage.get())) @@ -613,7 +634,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat QueryPipelineBuilder pipeline; - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, local_context->getViewSource(), SelectQueryOptions().ignoreAccessCheck()); pipeline = interpreter.buildQueryPipeline(); @@ -632,8 +653,8 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat /// and two-level aggregation is triggered). pipeline.addTransform(std::make_shared( pipeline.getHeader(), - context->getSettingsRef().min_insert_block_size_rows, - context->getSettingsRef().min_insert_block_size_bytes)); + context->getSettingsRef()[Setting::min_insert_block_size_rows], + context->getSettingsRef()[Setting::min_insert_block_size_bytes])); auto converting = ActionsDAG::makeConvertingActions( pipeline.getHeader().getColumnsWithTypeAndName(), @@ -663,9 +684,9 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat static void logQueryViews(std::list & views, ContextPtr context) { const auto & settings = context->getSettingsRef(); - const UInt64 min_query_duration = settings.log_queries_min_query_duration_ms.totalMilliseconds(); - const QueryViewsLogElement::ViewStatus min_status = settings.log_queries_min_type; - if (views.empty() || !settings.log_queries || !settings.log_query_views) + const UInt64 min_query_duration = settings[Setting::log_queries_min_query_duration_ms].totalMilliseconds(); + const QueryViewsLogElement::ViewStatus min_status = settings[Setting::log_queries_min_type]; + if (views.empty() || !settings[Setting::log_queries] || !settings[Setting::log_query_views]) return; for (auto & view : views) @@ -854,7 +875,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() if (!output.canPush()) return Status::PortFull; - bool materialized_views_ignore_errors = views_data->context->getSettingsRef().materialized_views_ignore_errors; + bool ignore_errors = views_data->context->getSettingsRef()[Setting::materialized_views_ignore_errors]; size_t num_finished = 0; size_t pos = 0; for (auto & input : inputs) @@ -880,7 +901,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() else statuses[i].exception = data.exception; - if (i == 0 && statuses[0].is_first && !materialized_views_ignore_errors) + if (i == 0 && statuses[0].is_first && !ignore_errors) { output.pushData(std::move(data)); return Status::PortFull; @@ -897,7 +918,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() if (!statuses.empty()) return Status::Ready; - if (any_exception && !materialized_views_ignore_errors) + if (any_exception && !ignore_errors) output.pushException(any_exception); output.finish(); @@ -925,8 +946,6 @@ static std::exception_ptr addStorageToException(std::exception_ptr ptr, const St void FinalizingViewsTransform::work() { - bool materialized_views_ignore_errors = views_data->context->getSettingsRef().materialized_views_ignore_errors; - size_t i = 0; for (auto & view : views_data->views) { @@ -941,7 +960,7 @@ void FinalizingViewsTransform::work() view.setException(addStorageToException(status.exception, view.table_id)); /// Exception will be ignored, it is saved here for the system.query_views_log - if (materialized_views_ignore_errors) + if (views_data->context->getSettingsRef()[Setting::materialized_views_ignore_errors]) tryLogException(view.exception, getLogger("PushingToViews"), "Cannot push to the storage, ignoring the error"); } else @@ -962,5 +981,4 @@ void FinalizingViewsTransform::work() statuses.clear(); } - } diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index c33e87815b1..4bb3b88886e 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool input_format_defaults_for_omitted_fields; + extern const SettingsUInt64 max_insert_block_size; +} namespace ErrorCodes { @@ -61,7 +66,8 @@ InputFormatPtr getInputFormatFromASTInsertQuery( : std::make_unique(); /// Create a source from input buffer using format from query - auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef().max_insert_block_size); + auto source + = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef()[Setting::max_insert_block_size]); source->addBuffer(std::move(input_buffer)); return source; } @@ -75,7 +81,7 @@ Pipe getSourceFromInputFormat( Pipe pipe(format); const auto * ast_insert_query = ast->as(); - if (context->getSettingsRef().input_format_defaults_for_omitted_fields && ast_insert_query->table_id && !input_function) + if (context->getSettingsRef()[Setting::input_format_defaults_for_omitted_fields] && ast_insert_query->table_id && !input_function) { StoragePtr storage = DatabaseCatalog::instance().getTable(ast_insert_query->table_id, context); auto metadata_snapshot = storage->getInMemoryMetadataPtr(); diff --git a/src/QueryPipeline/QueryPipeline.cpp b/src/QueryPipeline/QueryPipeline.cpp index c9c0bad7553..492ef7186b2 100644 --- a/src/QueryPipeline/QueryPipeline.cpp +++ b/src/QueryPipeline/QueryPipeline.cpp @@ -33,6 +33,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool rows_before_aggregation; +} namespace ErrorCodes { @@ -547,7 +551,7 @@ void QueryPipeline::complete(std::shared_ptr format) initRowsBeforeLimit(format.get()); for (const auto & context : resources.interpreter_context) { - if (context->getSettingsRef().rows_before_aggregation) + if (context->getSettingsRef()[Setting::rows_before_aggregation]) { initRowsBeforeAggregation(processors, format.get()); break; diff --git a/src/QueryPipeline/RemoteInserter.cpp b/src/QueryPipeline/RemoteInserter.cpp index 134c169e35f..9c0ffe221dd 100644 --- a/src/QueryPipeline/RemoteInserter.cpp +++ b/src/QueryPipeline/RemoteInserter.cpp @@ -13,6 +13,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsLogsLevel send_logs_level; +} + namespace ErrorCodes { extern const int UNEXPECTED_PACKET_FROM_SERVER; @@ -47,7 +52,7 @@ RemoteInserter::RemoteInserter( /// and will not consume Log packets. /// /// So that is why send_logs_level had been disabled here. - settings.send_logs_level = "none"; + settings[Setting::send_logs_level] = "none"; /** Send query and receive "header", that describes table structure. * Header is needed to know, what structure is required for blocks to be passed to 'write' method. */ diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 6f8b3931803..9e205d4df20 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -36,6 +36,15 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool enable_scalar_subquery_optimization; + extern const SettingsSeconds max_execution_time; + extern const SettingsSeconds max_estimated_execution_time; + extern const SettingsBool skip_unavailable_shards; + extern const SettingsOverflowMode timeout_overflow_mode; + extern const SettingsBool use_hedged_requests; +} namespace ErrorCodes { @@ -204,7 +213,7 @@ RemoteQueryExecutor::RemoteQueryExecutor( auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); #if defined(OS_LINUX) - if (current_settings.use_hedged_requests) + if (current_settings[Setting::use_hedged_requests]) { std::shared_ptr table_to_check = nullptr; if (main_table) @@ -395,7 +404,7 @@ void RemoteQueryExecutor::sendQueryUnlocked(ClientInfo::QueryKind query_kind, As established = false; sent_query = true; - if (settings.enable_scalar_subquery_optimization) + if (settings[Setting::enable_scalar_subquery_optimization]) sendScalars(); sendExternalTables(); } @@ -446,7 +455,7 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::read() { sendQuery(); - if (context->getSettingsRef().skip_unavailable_shards && (0 == connections->size())) + if (context->getSettingsRef()[Setting::skip_unavailable_shards] && (0 == connections->size())) return ReadResult(Block()); } @@ -822,9 +831,9 @@ void RemoteQueryExecutor::sendExternalTables() StreamLocalLimits limits; const auto & settings = context->getSettingsRef(); limits.mode = LimitsMode::LIMITS_TOTAL; - limits.speed_limits.max_execution_time = settings.max_execution_time; - limits.timeout_overflow_mode = settings.timeout_overflow_mode; - limits.speed_limits.max_estimated_execution_time = settings.max_estimated_execution_time; + limits.speed_limits.max_execution_time = settings[Setting::max_execution_time]; + limits.timeout_overflow_mode = settings[Setting::timeout_overflow_mode]; + limits.speed_limits.max_estimated_execution_time = settings[Setting::max_estimated_execution_time]; for (size_t i = 0; i < count; ++i) { @@ -921,7 +930,7 @@ void RemoteQueryExecutor::setProfileInfoCallback(ProfileInfoCallback callback) bool RemoteQueryExecutor::needToSkipUnavailableShard() const { - return context->getSettingsRef().skip_unavailable_shards && (0 == connections->size()); + return context->getSettingsRef()[Setting::skip_unavailable_shards] && (0 == connections->size()); } bool RemoteQueryExecutor::processParallelReplicaPacketIfAny() diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 9c8e0c6bf73..3aa651c89c9 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -59,6 +59,20 @@ using GRPCObsoleteTransportCompression = clickhouse::grpc::ObsoleteTransportComp namespace DB { +namespace Setting +{ + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsBool calculate_text_stack_trace; + extern const SettingsUInt64 interactive_delay; + extern const SettingsLogsLevel send_logs_level; + extern const SettingsString send_logs_source_regexp; + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; + extern const SettingsBool throw_if_no_data_to_insert; +} + namespace ErrorCodes { extern const int INVALID_CONFIG_PARAMETER; @@ -878,13 +892,13 @@ namespace /// Prepare for sending exceptions and logs. const Settings & settings = query_context->getSettingsRef(); - send_exception_with_stacktrace = settings.calculate_text_stack_trace; - const auto client_logs_level = settings.send_logs_level; + send_exception_with_stacktrace = settings[Setting::calculate_text_stack_trace]; + const auto client_logs_level = settings[Setting::send_logs_level]; if (client_logs_level != LogsLevel::none) { logs_queue = std::make_shared(); logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); - logs_queue->setSourceRegexp(settings.send_logs_source_regexp); + logs_queue->setSourceRegexp(settings[Setting::send_logs_source_regexp]); CurrentThread::attachInternalTextLogsQueue(logs_queue, client_logs_level); } @@ -897,15 +911,15 @@ namespace responder->setTransportCompression(*transport_compression); /// The interactive delay will be used to show progress. - interactive_delay = settings.interactive_delay; + interactive_delay = settings[Setting::interactive_delay]; query_context->setProgressCallback([this](const Progress & value) { return progress.incrementPiecewiseAtomically(value); }); /// Parse the query. query_text = std::move(*(query_info.mutable_query())); const char * begin = query_text.data(); const char * end = begin + query_text.size(); - ParserQuery parser(end, settings.allow_settings_after_format_in_insert); - ast = parseQuery(parser, begin, end, "", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ParserQuery parser(end, settings[Setting::allow_settings_after_format_in_insert]); + ast = parseQuery(parser, begin, end, "", settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); /// Choose input format. insert_query = ast->as(); @@ -994,7 +1008,7 @@ namespace else { const auto & settings = query_context->getSettingsRef(); - if (settings.throw_if_no_data_to_insert) + if (settings[Setting::throw_if_no_data_to_insert]) throw Exception(ErrorCodes::NO_DATA_TO_INSERT, "No data to insert"); else return; @@ -1084,7 +1098,7 @@ namespace assert(!pipeline); auto source - = query_context->getInputFormat(input_format, *read_buffer, header, query_context->getSettingsRef().max_insert_block_size); + = query_context->getInputFormat(input_format, *read_buffer, header, query_context->getSettingsRef()[Setting::max_insert_block_size]); pipeline = std::make_unique(std::move(source)); pipeline_executor = std::make_unique(*pipeline); @@ -1152,7 +1166,7 @@ namespace external_table_context->applySettingsChanges(settings_changes); } auto in = external_table_context->getInputFormat( - format, *buf, metadata_snapshot->getSampleBlock(), external_table_context->getSettingsRef().max_insert_block_size); + format, *buf, metadata_snapshot->getSampleBlock(), external_table_context->getSettingsRef()[Setting::max_insert_block_size]); QueryPipelineBuilder cur_pipeline; cur_pipeline.init(Pipe(std::move(in))); diff --git a/src/Server/HTTP/HTMLForm.cpp b/src/Server/HTTP/HTMLForm.cpp index 1abf9e5b83e..dfce2377941 100644 --- a/src/Server/HTTP/HTMLForm.cpp +++ b/src/Server/HTTP/HTMLForm.cpp @@ -19,6 +19,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 http_max_fields; + extern const SettingsUInt64 http_max_field_name_size; + extern const SettingsUInt64 http_max_field_value_size; +} namespace ErrorCodes { @@ -42,9 +48,9 @@ const int HTMLForm::UNKNOWN_CONTENT_LENGTH = -1; HTMLForm::HTMLForm(const Settings & settings) - : max_fields_number(settings.http_max_fields) - , max_field_name_size(settings.http_max_field_name_size) - , max_field_value_size(settings.http_max_field_value_size) + : max_fields_number(settings[Setting::http_max_fields]) + , max_field_name_size(settings[Setting::http_max_field_name_size]) + , max_field_value_size(settings[Setting::http_max_field_value_size]) , encoding(ENCODING_URL) { } diff --git a/src/Server/HTTP/authenticateUserByHTTP.cpp b/src/Server/HTTP/authenticateUserByHTTP.cpp index ac43bfd64c0..cbad91cc292 100644 --- a/src/Server/HTTP/authenticateUserByHTTP.cpp +++ b/src/Server/HTTP/authenticateUserByHTTP.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp b/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp index 48a3b39a3e7..e0694d8cc04 100644 --- a/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp +++ b/src/Server/HTTP/setReadOnlyIfHTTPMethodIdempotent.cpp @@ -7,6 +7,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 readonly; +} void setReadOnlyIfHTTPMethodIdempotent(ContextMutablePtr context, const String & http_method) { @@ -17,7 +21,7 @@ void setReadOnlyIfHTTPMethodIdempotent(ContextMutablePtr context, const String & /// readonly = 0 - any query is allowed, client can change any setting. /// readonly = 1 - only readonly queries are allowed, client can't change settings. /// readonly = 2 - only readonly queries are allowed, client can change any setting except 'readonly'. - if (context->getSettingsRef().readonly == 0) + if (context->getSettingsRef()[Setting::readonly] == 0) context->setSetting("readonly", 2); } } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index d2bc22e98cc..8a9ae05b355 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -54,6 +54,22 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool add_http_cors_header; + extern const SettingsBool cancel_http_readonly_queries_on_client_close; + extern const SettingsBool enable_http_compression; + extern const SettingsUInt64 http_headers_progress_interval_ms; + extern const SettingsUInt64 http_max_request_param_data_size; + extern const SettingsBool http_native_compression_disable_checksumming_on_decompress; + extern const SettingsUInt64 http_response_buffer_size; + extern const SettingsBool http_wait_end_of_query; + extern const SettingsBool http_write_exception_in_output_format; + extern const SettingsInt64 http_zlib_compression_level; + extern const SettingsUInt64 readonly; + extern const SettingsBool send_progress_in_http_headers; + extern const SettingsInt64 zstd_window_log_max; +} namespace ErrorCodes { @@ -249,18 +265,19 @@ void HTTPHandler::processQuery( /// At least, we should postpone sending of first buffer_size result bytes size_t buffer_size_total = std::max( - params.getParsed("buffer_size", context->getSettingsRef().http_response_buffer_size), + params.getParsed("buffer_size", context->getSettingsRef()[Setting::http_response_buffer_size]), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); /// If it is specified, the whole result will be buffered. /// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file. - bool buffer_until_eof = params.getParsed("wait_end_of_query", context->getSettingsRef().http_wait_end_of_query); + bool buffer_until_eof = params.getParsed("wait_end_of_query", context->getSettingsRef()[Setting::http_wait_end_of_query]); size_t buffer_size_http = DBMS_DEFAULT_BUFFER_SIZE; size_t buffer_size_memory = (buffer_size_total > buffer_size_http) ? buffer_size_total : 0; - bool enable_http_compression = params.getParsed("enable_http_compression", context->getSettingsRef().enable_http_compression); - Int64 http_zlib_compression_level = params.getParsed("http_zlib_compression_level", context->getSettingsRef().http_zlib_compression_level); + bool enable_http_compression = params.getParsed("enable_http_compression", context->getSettingsRef()[Setting::enable_http_compression]); + Int64 http_zlib_compression_level + = params.getParsed("http_zlib_compression_level", context->getSettingsRef()[Setting::http_zlib_compression_level]); used_output.out_holder = std::make_shared( @@ -273,12 +290,15 @@ void HTTPHandler::processQuery( if (client_supports_http_compression && enable_http_compression) { used_output.out_holder->setCompressionMethodHeader(http_response_compression_method); - used_output.wrap_compressed_holder = - wrapWriteBufferWithCompressionMethod( - used_output.out_holder.get(), - http_response_compression_method, - static_cast(http_zlib_compression_level), - 0, DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0, false); + used_output.wrap_compressed_holder = wrapWriteBufferWithCompressionMethod( + used_output.out_holder.get(), + http_response_compression_method, + static_cast(http_zlib_compression_level), + 0, + DBMS_DEFAULT_BUFFER_SIZE, + nullptr, + 0, + false); used_output.out = used_output.wrap_compressed_holder; } @@ -335,10 +355,11 @@ void HTTPHandler::processQuery( /// Request body can be compressed using algorithm specified in the Content-Encoding header. String http_request_compression_method_str = request.get("Content-Encoding", ""); - int zstd_window_log_max = static_cast(context->getSettingsRef().zstd_window_log_max); + int zstd_window_log_max = static_cast(context->getSettingsRef()[Setting::zstd_window_log_max]); auto in_post = wrapReadBufferWithCompressionMethod( wrapReadBufferReference(request.getStream()), - chooseCompressionMethod({}, http_request_compression_method_str), zstd_window_log_max); + chooseCompressionMethod({}, http_request_compression_method_str), + zstd_window_log_max); /// The data can also be compressed using incompatible internal algorithm. This is indicated by /// 'decompress' query parameter. @@ -431,18 +452,18 @@ void HTTPHandler::processQuery( const auto & query = getQuery(request, params, context); std::unique_ptr in_param = std::make_unique(query); - used_output.out_holder->setSendProgress(settings.send_progress_in_http_headers); - used_output.out_holder->setSendProgressInterval(settings.http_headers_progress_interval_ms); + used_output.out_holder->setSendProgress(settings[Setting::send_progress_in_http_headers]); + used_output.out_holder->setSendProgressInterval(settings[Setting::http_headers_progress_interval_ms]); /// If 'http_native_compression_disable_checksumming_on_decompress' setting is turned on, /// checksums of client data compressed with internal algorithm are not checked. - if (is_in_post_compressed && settings.http_native_compression_disable_checksumming_on_decompress) + if (is_in_post_compressed && settings[Setting::http_native_compression_disable_checksumming_on_decompress]) static_cast(*in_post_maybe_compressed).disableChecksumming(); /// Add CORS header if 'add_http_cors_header' setting is turned on send * in Access-Control-Allow-Origin /// Note that whether the header is added is determined by the settings, and we can only get the user settings after authentication. /// Once the authentication fails, the header can't be added. - if (settings.add_http_cors_header && !request.get("Origin", "").empty() && !config.has("http_options_response")) + if (settings[Setting::add_http_cors_header] && !request.get("Origin", "").empty() && !config.has("http_options_response")) used_output.out_holder->addHeaderCORS(true); auto append_callback = [my_context = context] (ProgressCallback callback) @@ -465,7 +486,7 @@ void HTTPHandler::processQuery( used_output.out_holder->onProgress(progress); }); - if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close) + if (settings[Setting::readonly] > 0 && settings[Setting::cancel_http_readonly_queries_on_client_close]) { append_callback([&context, &request](const Progress &) { @@ -496,9 +517,12 @@ void HTTPHandler::processQuery( response.add("X-ClickHouse-Timezone", *details.timezone); }; - auto handle_exception_in_output_format = [&](IOutputFormat & current_output_format, const String & format_name, const ContextPtr & context_, const std::optional & format_settings) + auto handle_exception_in_output_format = [&](IOutputFormat & current_output_format, + const String & format_name, + const ContextPtr & context_, + const std::optional & format_settings) { - if (settings.http_write_exception_in_output_format && current_output_format.supportsWritingException()) + if (settings[Setting::http_write_exception_in_output_format] && current_output_format.supportsWritingException()) { /// If wait_end_of_query=true in case of an exception all data written to output format during query execution will be /// ignored, so we cannot write exception message in current output format as it will be also ignored. @@ -876,7 +900,7 @@ void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, Conte WriteBufferFromOwnString value; const auto & settings = context->getSettingsRef(); - copyDataMaxBytes(body, value, settings.http_max_request_param_data_size); + copyDataMaxBytes(body, value, settings[Setting::http_max_request_param_data_size]); context->setQueryParameter("_request_body", value.str()); } } diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 5debc23f81a..e0c684b493b 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -39,6 +39,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool prefer_column_name_to_alias; + extern const SettingsSeconds receive_timeout; + extern const SettingsSeconds send_timeout; +} using namespace MySQLProtocol; using namespace MySQLProtocol::Generic; @@ -206,8 +212,8 @@ void MySQLHandler::run() session->setClientConnectionId(connection_id); const Settings & settings = server.context()->getSettingsRef(); - socket().setReceiveTimeout(settings.receive_timeout); - socket().setSendTimeout(settings.send_timeout); + socket().setReceiveTimeout(settings[Setting::receive_timeout]); + socket().setSendTimeout(settings[Setting::send_timeout]); in = std::make_shared(socket(), read_event); out = std::make_shared(socket(), write_event); @@ -479,12 +485,12 @@ void MySQLHandler::comQuery(ReadBuffer & payload, bool binary_protocol) /// --- Workaround for Bug 56173. Can be removed when the analyzer is on by default. auto settings = query_context->getSettingsCopy(); - settings.prefer_column_name_to_alias = true; + settings[Setting::prefer_column_name_to_alias] = true; query_context->setSettings(settings); /// Update timeouts - socket().setReceiveTimeout(settings.receive_timeout); - socket().setSendTimeout(settings.send_timeout); + socket().setReceiveTimeout(settings[Setting::receive_timeout]); + socket().setSendTimeout(settings[Setting::send_timeout]); CurrentThread::QueryScope query_scope{query_context}; diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index cde31b4c58a..5372f1dcddd 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -22,6 +22,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} namespace ErrorCodes { @@ -282,11 +289,13 @@ void PostgreSQLHandler::processQuery() const auto & settings = session->sessionContext()->getSettingsRef(); std::vector queries; - auto parse_res = splitMultipartQuery(query->query, queries, - settings.max_query_size, - settings.max_parser_depth, - settings.max_parser_backtracks, - settings.allow_settings_after_format_in_insert); + auto parse_res = splitMultipartQuery( + query->query, + queries, + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks], + settings[Setting::allow_settings_after_format_in_insert]); if (!parse_res.second) throw Exception(ErrorCodes::SYNTAX_ERROR, "Cannot parse and execute the following part of query: {}", String(parse_res.first)); diff --git a/src/Server/ProxyV1Handler.cpp b/src/Server/ProxyV1Handler.cpp index 5b33a119cef..d733454b2ab 100644 --- a/src/Server/ProxyV1Handler.cpp +++ b/src/Server/ProxyV1Handler.cpp @@ -8,6 +8,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds receive_timeout; +} namespace ErrorCodes { @@ -20,7 +24,7 @@ namespace ErrorCodes void ProxyV1Handler::run() { const auto & settings = server.context()->getSettingsRef(); - socket().setReceiveTimeout(settings.receive_timeout); + socket().setReceiveTimeout(settings[Setting::receive_timeout]); std::string word; bool eol; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 50963887752..dadaf614f92 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -66,6 +66,42 @@ using namespace std::literals; using namespace DB; +namespace DB +{ +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_codecs; + extern const SettingsBool allow_experimental_query_deduplication; + extern const SettingsBool allow_suspicious_codecs; + extern const SettingsBool async_insert; + extern const SettingsUInt64 async_insert_max_data_size; + extern const SettingsBool calculate_text_stack_trace; + extern const SettingsBool deduplicate_blocks_in_dependent_materialized_views; + extern const SettingsBool enable_deflate_qpl_codec; + extern const SettingsBool enable_zstd_qat_codec; + extern const SettingsUInt64 idle_connection_timeout; + extern const SettingsBool input_format_defaults_for_omitted_fields; + extern const SettingsUInt64 interactive_delay; + extern const SettingsBool low_cardinality_allow_in_native_format; + extern const SettingsString network_compression_method; + extern const SettingsInt64 network_zstd_compression_level; + extern const SettingsBool partial_result_on_first_cancel; + extern const SettingsUInt64 poll_interval; + extern const SettingsSeconds receive_timeout; + extern const SettingsLogsLevel send_logs_level; + extern const SettingsString send_logs_source_regexp; + extern const SettingsSeconds send_timeout; + extern const SettingsTimezone session_timezone; + extern const SettingsMilliseconds sleep_after_receiving_query_ms; + extern const SettingsMilliseconds sleep_in_send_data_ms; + extern const SettingsMilliseconds sleep_in_send_tables_status_ms; + extern const SettingsBool throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert; + extern const SettingsUInt64 unknown_packet_in_send_data; + extern const SettingsBool wait_for_async_insert; + extern const SettingsSeconds wait_for_async_insert_timeout; +} +} namespace CurrentMetrics { @@ -108,19 +144,6 @@ namespace DB::ErrorCodes namespace { -NameToNameMap convertToQueryParameters(const Settings & passed_params) -{ - NameToNameMap query_parameters; - for (const auto & param : passed_params) - { - std::string value; - ReadBufferFromOwnString buf(param.getValueString()); - readQuoted(value, buf); - query_parameters.emplace(param.getName(), value); - } - return query_parameters; -} - // This function corrects the wrong client_name from the old client. // Old clients 28.7 and some intermediate versions of 28.7 were sending different ClientInfo.client_name // "ClickHouse client" was sent with the hello message. @@ -442,13 +465,13 @@ void TCPHandler::runImpl() state.timeout_setter = std::make_unique(socket(), send_timeout, receive_timeout); /// Should we send internal logs to client? - const auto client_logs_level = query_context->getSettingsRef().send_logs_level; + const auto client_logs_level = query_context->getSettingsRef()[Setting::send_logs_level]; if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_LOGS && client_logs_level != LogsLevel::none) { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); - state.logs_queue->setSourceRegexp(query_context->getSettingsRef().send_logs_source_regexp); + state.logs_queue->setSourceRegexp(query_context->getSettingsRef()[Setting::send_logs_source_regexp]); CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); } if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_INCREMENTAL_PROFILE_EVENTS) @@ -485,7 +508,7 @@ void TCPHandler::runImpl() /// Send ColumnsDescription for input storage. if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA - && query_context->getSettingsRef().input_format_defaults_for_omitted_fields) + && query_context->getSettingsRef()[Setting::input_format_defaults_for_omitted_fields]) { sendTableColumns(metadata_snapshot->getColumns()); } @@ -815,15 +838,15 @@ void TCPHandler::runImpl() void TCPHandler::extractConnectionSettingsFromContext(const ContextPtr & context) { const auto & settings = context->getSettingsRef(); - send_exception_with_stack_trace = settings.calculate_text_stack_trace; - send_timeout = settings.send_timeout; - receive_timeout = settings.receive_timeout; - poll_interval = settings.poll_interval; - idle_connection_timeout = settings.idle_connection_timeout; - interactive_delay = settings.interactive_delay; - sleep_in_send_tables_status = settings.sleep_in_send_tables_status_ms; - unknown_packet_in_send_data = settings.unknown_packet_in_send_data; - sleep_after_receiving_query = settings.sleep_after_receiving_query_ms; + send_exception_with_stack_trace = settings[Setting::calculate_text_stack_trace]; + send_timeout = settings[Setting::send_timeout]; + receive_timeout = settings[Setting::receive_timeout]; + poll_interval = settings[Setting::poll_interval]; + idle_connection_timeout = settings[Setting::idle_connection_timeout]; + interactive_delay = settings[Setting::interactive_delay]; + sleep_in_send_tables_status = settings[Setting::sleep_in_send_tables_status_ms]; + unknown_packet_in_send_data = settings[Setting::unknown_packet_in_send_data]; + sleep_after_receiving_query = settings[Setting::sleep_after_receiving_query_ms]; } @@ -833,7 +856,8 @@ bool TCPHandler::readDataNext() /// Poll interval should not be greater than receive_timeout constexpr UInt64 min_timeout_us = 5000; // 5 ms - UInt64 timeout_us = std::max(min_timeout_us, std::min(poll_interval * 1000000, static_cast(receive_timeout.totalMicroseconds()))); + UInt64 timeout_us + = std::max(min_timeout_us, std::min(poll_interval * 1000000, static_cast(receive_timeout.totalMicroseconds()))); bool read_ok = false; /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. @@ -916,7 +940,7 @@ void TCPHandler::startInsertQuery() if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA) { const auto & table_id = query_context->getInsertionTable(); - if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields) + if (query_context->getSettingsRef()[Setting::input_format_defaults_for_omitted_fields]) { if (!table_id.empty()) { @@ -936,7 +960,7 @@ AsynchronousInsertQueue::PushResult TCPHandler::processAsyncInsertQuery(Asynchro using PushResult = AsynchronousInsertQueue::PushResult; startInsertQuery(); - Squashing squashing(state.input_header, 0, query_context->getSettingsRef().async_insert_max_data_size); + Squashing squashing(state.input_header, 0, query_context->getSettingsRef()[Setting::async_insert_max_data_size]); while (readDataNext()) { @@ -994,7 +1018,7 @@ void TCPHandler::processInsertQuery() auto * insert_queue = query_context->tryGetAsynchronousInsertQueue(); const auto & insert_query = assert_cast(*state.parsed_query); - bool async_insert_enabled = settings.async_insert; + bool async_insert_enabled = settings[Setting::async_insert]; if (insert_query.table_id) if (auto table = DatabaseCatalog::instance().tryGetTable(insert_query.table_id, query_context)) async_insert_enabled |= table->areAsynchronousInsertsEnabled(); @@ -1010,8 +1034,8 @@ void TCPHandler::processInsertQuery() /// The process of forming such blocks is not deteministic so each time we retry mini-INSERTs the resulting /// block may be concatenated differently. /// That's why deduplication in dependent Materialized Views doesn't make sense in presence of async INSERTs. - if (settings.throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert && - settings.deduplicate_blocks_in_dependent_materialized_views) + if (settings[Setting::throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert] + && settings[Setting::deduplicate_blocks_in_dependent_materialized_views]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Deduplication in dependent materialized view cannot work together with async inserts. "\ "Please disable either `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting."); @@ -1021,9 +1045,9 @@ void TCPHandler::processInsertQuery() { /// Reset pipeline because it may hold write lock for some storages. state.io.pipeline.reset(); - if (settings.wait_for_async_insert) + if (settings[Setting::wait_for_async_insert]) { - size_t timeout_ms = settings.wait_for_async_insert_timeout.totalMilliseconds(); + size_t timeout_ms = settings[Setting::wait_for_async_insert_timeout].totalMilliseconds(); auto wait_status = result.future.wait_for(std::chrono::milliseconds(timeout_ms)); if (wait_status == std::future_status::deferred) @@ -1064,7 +1088,7 @@ void TCPHandler::processOrdinaryQuery() { auto & pipeline = state.io.pipeline; - if (query_context->getSettingsRef().allow_experimental_query_deduplication) + if (query_context->getSettingsRef()[Setting::allow_experimental_query_deduplication]) { std::lock_guard lock(out_mutex); sendPartUUIDs(); @@ -1379,7 +1403,7 @@ void TCPHandler::sendTimezone() if (client_tcp_protocol_version < DBMS_MIN_PROTOCOL_VERSION_WITH_TIMEZONE_UPDATES) return; - const String & tz = query_context->getSettingsRef().session_timezone.value; + const String & tz = query_context->getSettingsRef()[Setting::session_timezone].value; LOG_DEBUG(log, "TCPHandler::sendTimezone(): {}", tz); writeVarUInt(Protocol::Server::TimezoneUpdate, *out); @@ -2005,9 +2029,8 @@ void TCPHandler::receiveQuery() /// Analyzer became Beta in 24.3 and started to be enabled by default. /// We have to disable it for ourselves to make sure we don't have different settings on /// different servers. - if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY - && client_info.getVersionNumber() < VersionNumber(23, 3, 0) - && !passed_settings.allow_experimental_analyzer.changed) + if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && client_info.getVersionNumber() < VersionNumber(23, 3, 0) + && !passed_settings[Setting::allow_experimental_analyzer].changed) passed_settings.set("allow_experimental_analyzer", false); auto settings_changes = passed_settings.changes(); @@ -2035,7 +2058,7 @@ void TCPHandler::receiveQuery() /// so we have to apply the changes first. query_context->setCurrentQueryId(state.query_id); - query_context->addQueryParameters(convertToQueryParameters(passed_params)); + query_context->addQueryParameters(passed_params.toNameToNameMap()); /// For testing hedged requests if (unlikely(sleep_after_receiving_query.totalMilliseconds())) @@ -2197,14 +2220,20 @@ void TCPHandler::initBlockOutput(const Block & block) const Settings & query_settings = query_context->getSettingsRef(); if (!state.maybe_compressed_out) { - std::string method = Poco::toUpper(query_settings.network_compression_method.toString()); + std::string method = Poco::toUpper(query_settings[Setting::network_compression_method].toString()); std::optional level; if (method == "ZSTD") - level = query_settings.network_zstd_compression_level; + level = query_settings[Setting::network_zstd_compression_level]; if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_deflate_qpl_codec, query_settings.enable_zstd_qat_codec); + CompressionCodecFactory::instance().validateCodec( + method, + level, + !query_settings[Setting::allow_suspicious_codecs], + query_settings[Setting::allow_experimental_codecs], + query_settings[Setting::enable_deflate_qpl_codec], + query_settings[Setting::enable_zstd_qat_codec]); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); @@ -2218,7 +2247,7 @@ void TCPHandler::initBlockOutput(const Block & block) client_tcp_protocol_version, block.cloneEmpty(), std::nullopt, - !query_settings.low_cardinality_allow_in_native_format); + !query_settings[Setting::low_cardinality_allow_in_native_format]); } } @@ -2229,11 +2258,7 @@ void TCPHandler::initLogsBlockOutput(const Block & block) /// Use uncompressed stream since log blocks usually contain only one row const Settings & query_settings = query_context->getSettingsRef(); state.logs_block_out = std::make_unique( - *out, - client_tcp_protocol_version, - block.cloneEmpty(), - std::nullopt, - !query_settings.low_cardinality_allow_in_native_format); + *out, client_tcp_protocol_version, block.cloneEmpty(), std::nullopt, !query_settings[Setting::low_cardinality_allow_in_native_format]); } } @@ -2244,11 +2269,7 @@ void TCPHandler::initProfileEventsBlockOutput(const Block & block) { const Settings & query_settings = query_context->getSettingsRef(); state.profile_events_block_out = std::make_unique( - *out, - client_tcp_protocol_version, - block.cloneEmpty(), - std::nullopt, - !query_settings.low_cardinality_allow_in_native_format); + *out, client_tcp_protocol_version, block.cloneEmpty(), std::nullopt, !query_settings[Setting::low_cardinality_allow_in_native_format]); } } @@ -2260,7 +2281,7 @@ void TCPHandler::decreaseCancellationStatus(const std::string & log_message) if (query_context) { const auto & settings = query_context->getSettingsRef(); - partial_result_on_first_cancel = settings.partial_result_on_first_cancel; + partial_result_on_first_cancel = settings[Setting::partial_result_on_first_cancel]; } if (partial_result_on_first_cancel && state.cancellation_status == CancellationStatus::NOT_CANCELLED) @@ -2340,7 +2361,7 @@ void TCPHandler::sendData(const Block & block) writeVarUInt(Protocol::Server::Data, *out); /// For testing hedged requests - if (block.rows() > 0 && query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()) + if (block.rows() > 0 && query_context->getSettingsRef()[Setting::sleep_in_send_data_ms].totalMilliseconds()) { /// This strange sequence is needed in case of chunked protocol is enabled, in order for client not to /// hang on receiving of at least packet type - chunk will not be processed unless either chunk footer @@ -2350,7 +2371,7 @@ void TCPHandler::sendData(const Block & block) /// Send external table name (empty name is the main table) writeStringBinary("", *out); out->next(); - std::chrono::milliseconds ms(query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()); + std::chrono::milliseconds ms(query_context->getSettingsRef()[Setting::sleep_in_send_data_ms].totalMilliseconds()); std::this_thread::sleep_for(ms); } else diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 68778243371..460d74e68bf 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -44,6 +44,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_codecs; + extern const SettingsBool allow_suspicious_codecs; + extern const SettingsBool allow_suspicious_ttl_expressions; + extern const SettingsBool enable_deflate_qpl_codec; + extern const SettingsBool enable_zstd_qat_codec; + extern const SettingsBool flatten_nested; +} namespace ErrorCodes { @@ -490,7 +500,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.ttl = ttl; - if (context->getSettingsRef().flatten_nested) + if (context->getSettingsRef()[Setting::flatten_nested]) { StorageInMemoryMetadata temporary_metadata; temporary_metadata.columns.add(column, /*after_column*/ "", /*first*/ true); @@ -797,7 +807,8 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) } else if (type == MODIFY_TTL) { - metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl, metadata.columns, context, metadata.primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions); + metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + ttl, metadata.columns, context, metadata.primary_key, context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]); } else if (type == REMOVE_TTL) { @@ -808,7 +819,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) metadata.select = SelectQueryDescription::getSelectQueryFromASTForMatView(select, metadata.refresh != nullptr, context); Block as_select_sample; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(select->clone(), context); } @@ -1249,13 +1260,18 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context metadata_copy.column_ttls_by_name.clear(); for (const auto & [name, ast] : column_ttl_asts) { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, metadata_copy.columns, context, metadata_copy.primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions); + auto new_ttl_entry = TTLDescription::getTTLFromAST( + ast, metadata_copy.columns, context, metadata_copy.primary_key, context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]); metadata_copy.column_ttls_by_name[name] = new_ttl_entry; } if (metadata_copy.table_ttl.definition_ast != nullptr) metadata_copy.table_ttl = TTLTableDescription::getTTLForTableFromAST( - metadata_copy.table_ttl.definition_ast, metadata_copy.columns, context, metadata_copy.primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions); + metadata_copy.table_ttl.definition_ast, + metadata_copy.columns, + context, + metadata_copy.primary_key, + context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]); metadata = std::move(metadata_copy); } @@ -1365,11 +1381,12 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { const auto & settings = context->getSettingsRef(); CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - command.codec, command.data_type, - !settings.allow_suspicious_codecs, - settings.allow_experimental_codecs, - settings.enable_deflate_qpl_codec, - settings.enable_zstd_qat_codec); + command.codec, + command.data_type, + !settings[Setting::allow_suspicious_codecs], + settings[Setting::allow_experimental_codecs], + settings[Setting::enable_deflate_qpl_codec], + settings[Setting::enable_zstd_qat_codec]); } all_columns.add(ColumnDescription(column_name, command.data_type)); @@ -1395,7 +1412,13 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { if (all_columns.hasAlias(column_name)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec, context->getSettingsRef().enable_zstd_qat_codec); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( + command.codec, + command.data_type, + !context->getSettingsRef()[Setting::allow_suspicious_codecs], + context->getSettingsRef()[Setting::allow_experimental_codecs], + context->getSettingsRef()[Setting::enable_deflate_qpl_codec], + context->getSettingsRef()[Setting::enable_zstd_qat_codec]); } auto column_default = all_columns.getDefault(column_name); if (column_default) diff --git a/src/Storages/Cache/ExternalDataSourceCache.cpp b/src/Storages/Cache/ExternalDataSourceCache.cpp index 8c778fd511a..d1fe679520e 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.cpp +++ b/src/Storages/Cache/ExternalDataSourceCache.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 625c64128e7..e55eb01ae74 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -16,6 +16,10 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool distributed_insert_skip_read_only_replicas; +} namespace ErrorCodes { @@ -242,7 +246,7 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); - auto result = parent.pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = parent.pool->getValidTryResult(results, insert_settings[Setting::distributed_insert_skip_read_only_replicas]); connection = std::move(result.entry); compression_expected = connection->getCompression() == Protocol::Compression::Enable; @@ -301,7 +305,7 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); - auto result = parent.pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = parent.pool->getValidTryResult(results, insert_settings[Setting::distributed_insert_skip_read_only_replicas]); auto connection = std::move(result.entry); bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 7f368102dfd..c517a055044 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -48,6 +48,15 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool distributed_insert_skip_read_only_replicas; + extern const SettingsSeconds distributed_replica_error_half_life; + extern const SettingsUInt64 distributed_replica_error_cap; + extern const SettingsLoadBalancing load_balancing; + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_rows; +} namespace ErrorCodes { @@ -114,8 +123,8 @@ DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue( , should_batch_inserts(storage.getDistributedSettingsRef().background_insert_batch) , split_batch_on_failure(storage.getDistributedSettingsRef().background_insert_split_batch_on_failure) , dir_fsync(storage.getDistributedSettingsRef().fsync_directories) - , min_batched_block_size_rows(storage.getContext()->getSettingsRef().min_insert_block_size_rows) - , min_batched_block_size_bytes(storage.getContext()->getSettingsRef().min_insert_block_size_bytes) + , min_batched_block_size_rows(storage.getContext()->getSettingsRef()[Setting::min_insert_block_size_rows]) + , min_batched_block_size_bytes(storage.getContext()->getSettingsRef()[Setting::min_insert_block_size_bytes]) , current_batch_file_path(path + "current_batch.txt") , pending_files(std::numeric_limits::max()) , default_sleep_time(storage.getDistributedSettingsRef().background_insert_sleep_time_ms.totalMilliseconds()) @@ -286,10 +295,11 @@ ConnectionPoolWithFailoverPtr DistributedAsyncInsertDirectoryQueue::createPool(c auto pools = createPoolsForAddresses(addresses, pool_factory, storage.log); const auto & settings = storage.getContext()->getSettingsRef(); - return std::make_shared(std::move(pools), - settings.load_balancing, - settings.distributed_replica_error_half_life.totalSeconds(), - settings.distributed_replica_error_cap); + return std::make_shared( + std::move(pools), + settings[Setting::load_balancing], + settings[Setting::distributed_replica_error_half_life].totalSeconds(), + settings[Setting::distributed_replica_error_cap]); } bool DistributedAsyncInsertDirectoryQueue::hasPendingFiles() const @@ -415,7 +425,7 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path, auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); auto results = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); - auto result = pool->getValidTryResult(results, insert_settings.distributed_insert_skip_read_only_replicas); + auto result = pool->getValidTryResult(results, insert_settings[Setting::distributed_insert_skip_read_only_replicas]); auto connection = std::move(result.entry); LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", @@ -467,7 +477,7 @@ struct DistributedAsyncInsertDirectoryQueue::BatchHeader Block header; BatchHeader(Settings settings_, String query_, ClientInfo client_info_, Block header_) - : settings(std::move(settings_)) + : settings(settings_) , query(std::move(query_)) , client_info(std::move(client_info_)) , header(std::move(header_)) diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h index ba4d264f967..972e6b3143a 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h @@ -1,12 +1,13 @@ #pragma once -#include -#include +#include #include +#include +#include #include #include -#include -#include +#include +#include namespace CurrentMetrics { class Increment; } diff --git a/src/Storages/Distributed/DistributedSettings.h b/src/Storages/Distributed/DistributedSettings.h index efbd6900b07..c6ad9ab6fa4 100644 --- a/src/Storages/Distributed/DistributedSettings.h +++ b/src/Storages/Distributed/DistributedSettings.h @@ -17,7 +17,7 @@ class ASTStorage; #define LIST_OF_DISTRIBUTED_SETTINGS(M, ALIAS) \ M(Bool, fsync_after_insert, false, "Do fsync for every inserted. Will decreases performance of inserts (only for background INSERT, i.e. distributed_foreground_insert=false)", 0) \ M(Bool, fsync_directories, false, "Do fsync for temporary directory (that is used for background INSERT only) after all part operations (writes, renames, etc.).", 0) \ - /** This is the distributed version of the skip_unavailable_shards setting available in src/Core/Settings.h */ \ + /** This is the distributed version of the skip_unavailable_shards setting available in src/Core/Settings.cpp */ \ M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards. Shard is marked as unavailable when: 1) The shard cannot be reached due to a connection failure. 2) Shard is unresolvable through DNS. 3) Table does not exist on the shard.", 0) \ /** Inserts settings. */ \ M(UInt64, bytes_to_throw_insert, 0, "If more than this number of compressed bytes will be pending for background INSERT, an exception will be thrown. 0 - do not throw.", 0) \ diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index f01ea10065c..5bc3fcc5be3 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -54,6 +54,26 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_codecs; + extern const SettingsBool allow_suspicious_codecs; + extern const SettingsMilliseconds distributed_background_insert_sleep_time_ms; + extern const SettingsBool distributed_insert_skip_read_only_replicas; + extern const SettingsBool enable_deflate_qpl_codec; + extern const SettingsBool enable_zstd_qat_codec; + extern const SettingsBool insert_allow_materialized_columns; + extern const SettingsBool insert_distributed_one_random_shard; + extern const SettingsUInt64 insert_shard_id; + extern const SettingsUInt64 max_distributed_connections; + extern const SettingsUInt64 max_distributed_depth; + extern const SettingsUInt64 max_network_bandwidth; + extern const SettingsUInt64 max_network_bytes; + extern const SettingsString network_compression_method; + extern const SettingsInt64 network_zstd_compression_level; + extern const SettingsBool prefer_localhost_replica; + extern const SettingsBool use_compact_format_in_distributed_parts_names; +} namespace ErrorCodes { @@ -122,16 +142,16 @@ DistributedSink::DistributedSink( , query_string(queryToString(query_ast)) , cluster(cluster_) , insert_sync(insert_sync_) - , allow_materialized(context->getSettingsRef().insert_allow_materialized_columns) + , allow_materialized(context->getSettingsRef()[Setting::insert_allow_materialized_columns]) , insert_timeout(insert_timeout_) , columns_to_send(columns_to_send_.begin(), columns_to_send_.end()) , log(getLogger("DistributedSink")) { const auto & settings = context->getSettingsRef(); - if (settings.max_distributed_depth && context->getClientInfo().distributed_depth >= settings.max_distributed_depth) + if (settings[Setting::max_distributed_depth] && context->getClientInfo().distributed_depth >= settings[Setting::max_distributed_depth]) throw Exception(ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH, "Maximum distributed depth exceeded"); context->increaseDistributedDepth(); - random_shard_insert = settings.insert_distributed_one_random_shard && !storage.has_sharding_key; + random_shard_insert = settings[Setting::insert_distributed_one_random_shard] && !storage.has_sharding_key; } @@ -235,13 +255,13 @@ void DistributedSink::initWritingJobs(const Block & first_block, size_t start, s auto & shard_jobs = per_shard_jobs[shard_index]; /// If hasInternalReplication, than prefer local replica (if !prefer_localhost_replica) - if (!shard_info.hasInternalReplication() || !shard_info.isLocal() || !settings.prefer_localhost_replica) + if (!shard_info.hasInternalReplication() || !shard_info.isLocal() || !settings[Setting::prefer_localhost_replica]) { const auto & replicas = addresses_with_failovers[shard_index]; for (size_t replica_index : collections::range(0, replicas.size())) { - if (!replicas[replica_index].is_local || !settings.prefer_localhost_replica) + if (!replicas[replica_index].is_local || !settings[Setting::prefer_localhost_replica]) { shard_jobs.replicas_jobs.emplace_back(shard_index, replica_index, false, first_block); ++remote_jobs_count; @@ -252,7 +272,7 @@ void DistributedSink::initWritingJobs(const Block & first_block, size_t start, s } } - if (shard_info.isLocal() && settings.prefer_localhost_replica) + if (shard_info.isLocal() && settings[Setting::prefer_localhost_replica]) { shard_jobs.replicas_jobs.emplace_back(shard_index, 0, true, first_block); ++local_jobs_count; @@ -362,7 +382,7 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si if (rows == 0) return; - if (!job.is_local_job || !settings.prefer_localhost_replica) + if (!job.is_local_job || !settings[Setting::prefer_localhost_replica]) { if (!job.executor) { @@ -377,7 +397,7 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si /// NOTE: INSERT will also take into account max_replica_delay_for_distributed_queries /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default) auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); - auto result = shard_info.pool->getValidTryResult(results, settings.distributed_insert_skip_read_only_replicas); + auto result = shard_info.pool->getValidTryResult(results, settings[Setting::distributed_insert_skip_read_only_replicas]); job.connection_entry = std::move(result.entry); } else @@ -460,10 +480,10 @@ void DistributedSink::writeSync(const Block & block) size_t start = 0; size_t end = shards_info.size(); - if (settings.insert_shard_id) + if (settings[Setting::insert_shard_id]) { - start = settings.insert_shard_id - 1; - end = settings.insert_shard_id; + start = settings[Setting::insert_shard_id] - 1; + end = settings[Setting::insert_shard_id]; } if (!pool) @@ -472,17 +492,17 @@ void DistributedSink::writeSync(const Block & block) initWritingJobs(block_to_send, start, end); size_t jobs_count = random_shard_insert ? 1 : (remote_jobs_count + local_jobs_count); - size_t max_threads = std::min(settings.max_distributed_connections, jobs_count); + size_t max_threads = std::min(settings[Setting::max_distributed_connections], jobs_count); pool.emplace( CurrentMetrics::DistributedInsertThreads, CurrentMetrics::DistributedInsertThreadsActive, CurrentMetrics::DistributedInsertThreadsScheduled, max_threads, max_threads, jobs_count); - if (!throttler && (settings.max_network_bandwidth || settings.max_network_bytes)) + if (!throttler && (settings[Setting::max_network_bandwidth] || settings[Setting::max_network_bytes])) { - throttler = std::make_shared(settings.max_network_bandwidth, settings.max_network_bytes, - "Network bandwidth limit for a query exceeded."); + throttler = std::make_shared( + settings[Setting::max_network_bandwidth], settings[Setting::max_network_bytes], "Network bandwidth limit for a query exceeded."); } watch.restart(); @@ -694,14 +714,13 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) if (shard_info.hasInternalReplication()) { - if (shard_info.isLocal() && settings.prefer_localhost_replica) + if (shard_info.isLocal() && settings[Setting::prefer_localhost_replica]) /// Prefer insert into current instance directly writeToLocal(shard_info, block_to_send, shard_info.getLocalNodeCount()); else { const auto & path = shard_info.insertPathForInternalReplication( - settings.prefer_localhost_replica, - settings.use_compact_format_in_distributed_parts_names); + settings[Setting::prefer_localhost_replica], settings[Setting::use_compact_format_in_distributed_parts_names]); if (path.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Directory name for async inserts is empty"); writeToShard(shard_info, block_to_send, {path}); @@ -709,13 +728,13 @@ void DistributedSink::writeAsyncImpl(const Block & block, size_t shard_id) } else { - if (shard_info.isLocal() && settings.prefer_localhost_replica) + if (shard_info.isLocal() && settings[Setting::prefer_localhost_replica]) writeToLocal(shard_info, block_to_send, shard_info.getLocalNodeCount()); std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) - if (!address.is_local || !settings.prefer_localhost_replica) - dir_names.push_back(address.toFullString(settings.use_compact_format_in_distributed_parts_names)); + if (!address.is_local || !settings[Setting::prefer_localhost_replica]) + dir_names.push_back(address.toFullString(settings[Setting::use_compact_format_in_distributed_parts_names])); if (!dir_names.empty()) writeToShard(shard_info, block_to_send, dir_names); @@ -769,13 +788,19 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const bool fsync = distributed_settings.fsync_after_insert; bool dir_fsync = distributed_settings.fsync_directories; - std::string compression_method = Poco::toUpper(settings.network_compression_method.toString()); + std::string compression_method = Poco::toUpper(settings[Setting::network_compression_method].toString()); std::optional compression_level; if (compression_method == "ZSTD") - compression_level = settings.network_zstd_compression_level; + compression_level = settings[Setting::network_zstd_compression_level]; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_deflate_qpl_codec, settings.enable_zstd_qat_codec); + CompressionCodecFactory::instance().validateCodec( + compression_method, + compression_level, + !settings[Setting::allow_suspicious_codecs], + settings[Setting::allow_experimental_codecs], + settings[Setting::enable_deflate_qpl_codec], + settings[Setting::enable_zstd_qat_codec]); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions @@ -798,7 +823,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const return guard; }; - auto sleep_ms = context->getSettingsRef().distributed_background_insert_sleep_time_ms.totalMilliseconds(); + auto sleep_ms = context->getSettingsRef()[Setting::distributed_background_insert_sleep_time_ms].totalMilliseconds(); size_t file_size; auto it = dir_names.begin(); diff --git a/src/Storages/FileLog/FileLogSettings.h b/src/Storages/FileLog/FileLogSettings.h index 9e64b80f57a..fd20dea702a 100644 --- a/src/Storages/FileLog/FileLogSettings.h +++ b/src/Storages/FileLog/FileLogSettings.h @@ -1,8 +1,8 @@ #pragma once #include -#include - +#include +#include namespace DB { diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 0f9bd8b6ff9..9222084ca0e 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -33,6 +34,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsMilliseconds stream_poll_timeout_ms; + extern const SettingsBool use_concurrency_control; +} namespace ErrorCodes { @@ -577,20 +585,20 @@ StorageFileLog::ReadMetadataResult StorageFileLog::readMetadata(const String & f size_t StorageFileLog::getMaxBlockSize() const { return filelog_settings->max_block_size.changed ? filelog_settings->max_block_size.value - : getContext()->getSettingsRef().max_insert_block_size.value; + : getContext()->getSettingsRef()[Setting::max_insert_block_size].value; } size_t StorageFileLog::getPollMaxBatchSize() const { size_t batch_size = filelog_settings->poll_max_batch_size.changed ? filelog_settings->poll_max_batch_size.value - : getContext()->getSettingsRef().max_block_size.value; + : getContext()->getSettingsRef()[Setting::max_block_size].value; return std::min(batch_size, getMaxBlockSize()); } size_t StorageFileLog::getPollTimeoutMillisecond() const { return filelog_settings->poll_timeout_ms.changed ? filelog_settings->poll_timeout_ms.totalMilliseconds() - : getContext()->getSettingsRef().stream_poll_timeout_ms.totalMilliseconds(); + : getContext()->getSettingsRef()[Setting::stream_poll_timeout_ms].totalMilliseconds(); } bool StorageFileLog::checkDependencies(const StorageID & table_id) @@ -777,7 +785,7 @@ bool StorageFileLog::streamToViews() { block_io.pipeline.complete(std::move(input)); block_io.pipeline.setNumThreads(max_streams_number); - block_io.pipeline.setConcurrencyControl(new_context->getSettingsRef().use_concurrency_control); + block_io.pipeline.setConcurrencyControl(new_context->getSettingsRef()[Setting::use_concurrency_control]); block_io.pipeline.setProgressCallback([&](const Progress & progress) { rows += progress.read_rows.load(); }); CompletedPipelineExecutor executor(block_io.pipeline); executor.execute(); diff --git a/src/Storages/Hive/HiveSettings.h b/src/Storages/Hive/HiveSettings.h index cbef19b3fa7..90156007f42 100644 --- a/src/Storages/Hive/HiveSettings.h +++ b/src/Storages/Hive/HiveSettings.h @@ -6,7 +6,8 @@ #include #include -#include +#include +#include namespace DB { diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index ea2e9e3eece..dcae5cad985 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,16 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool input_format_parquet_case_insensitive_column_matching; + extern const SettingsBool input_format_orc_case_insensitive_column_matching; + extern const SettingsUInt64 max_block_size; + extern const SettingsInt64 max_partitions_to_read; + extern const SettingsMaxThreads max_threads; + extern const SettingsBool use_local_cache_for_remote_storage; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -267,7 +278,7 @@ public: /// Use local cache for remote storage if enabled. std::unique_ptr remote_read_buf; if (ExternalDataSourceCache::instance().isInitialized() - && getContext()->getSettingsRef().use_local_cache_for_remote_storage) + && getContext()->getSettingsRef()[Setting::use_local_cache_for_remote_storage]) { size_t buff_size = raw_read_buf->internalBuffer().size(); if (buff_size == 0) @@ -289,7 +300,13 @@ public: read_buf = std::move(remote_read_buf); auto input_format = FormatFactory::instance().getInput( - format, *read_buf, to_read_block, getContext(), max_block_size, updateFormatSettings(current_file), /* max_parsing_threads */ 1); + format, + *read_buf, + to_read_block, + getContext(), + max_block_size, + updateFormatSettings(current_file), + /* max_parsing_threads */ 1); Pipe pipe(input_format); if (columns_description.hasDefaults()) @@ -321,7 +338,7 @@ public: Chunk generateChunkFromMetadata() { - size_t num_rows = std::min(current_file_remained_rows, UInt64(getContext()->getSettingsRef().max_block_size)); + size_t num_rows = std::min(current_file_remained_rows, UInt64(getContext()->getSettingsRef()[Setting::max_block_size])); current_file_remained_rows -= num_rows; Block source_block; @@ -626,7 +643,7 @@ HiveFiles StorageHive::collectHiveFilesFromPartition( buffer, partition_key_expr->getSampleBlock(), getContext(), - getContext()->getSettingsRef().max_block_size, + getContext()->getSettingsRef()[Setting::max_block_size], std::nullopt, /* max_parsing_threads */ 1); auto pipeline = QueryPipeline(std::move(format)); @@ -788,12 +805,7 @@ public: LoggerPtr log_, size_t max_block_size_, size_t num_streams_) - : SourceStepWithFilter( - DataStream{.header = std::move(header)}, - column_names_, - query_info_, - storage_snapshot_, - context_) + : SourceStepWithFilter(DataStream{.header = std::move(header)}, column_names_, query_info_, storage_snapshot_, context_) , storage(std::move(storage_)) , sources_info(std::move(sources_info_)) , builder(std::move(builder_)) @@ -861,9 +873,9 @@ void StorageHive::read( auto case_insensitive_matching = [&]() -> bool { if (format_name == "Parquet") - return settings.input_format_parquet_case_insensitive_column_matching; + return settings[Setting::input_format_parquet_case_insensitive_column_matching]; else if (format_name == "ORC") - return settings.input_format_orc_case_insensitive_column_matching; + return settings[Setting::input_format_orc_case_insensitive_column_matching]; return false; }; Block sample_block; @@ -964,10 +976,14 @@ HiveFiles StorageHive::collectHiveFiles( /// Hive files to collect HiveFiles hive_files; Int64 hit_parttions_num = 0; - Int64 hive_max_query_partitions = context_->getSettingsRef().max_partitions_to_read; + Int64 hive_max_query_partitions = context_->getSettingsRef()[Setting::max_partitions_to_read]; /// Mutext to protect hive_files, which maybe appended in multiple threads std::mutex hive_files_mutex; - ThreadPool pool{CurrentMetrics::StorageHiveThreads, CurrentMetrics::StorageHiveThreadsActive, CurrentMetrics::StorageHiveThreadsScheduled, max_threads}; + ThreadPool pool{ + CurrentMetrics::StorageHiveThreads, + CurrentMetrics::StorageHiveThreadsActive, + CurrentMetrics::StorageHiveThreadsScheduled, + max_threads}; if (!partitions.empty()) { for (const auto & partition : partitions) @@ -1053,13 +1069,7 @@ StorageHive::totalRowsImpl(const Settings & settings, const ActionsDAG * filter_ auto hive_table_metadata = hive_metastore_client->getTableMetadata(hive_database, hive_table); HDFSBuilderWrapper builder = createHDFSBuilder(hdfs_namenode_url, getContext()->getGlobalContext()->getConfigRef()); HDFSFSPtr fs = createHDFSFS(builder.get()); - HiveFiles hive_files = collectHiveFiles( - settings.max_threads, - filter_actions_dag, - hive_table_metadata, - fs, - context_, - prune_level); + HiveFiles hive_files = collectHiveFiles(settings[Setting::max_threads], filter_actions_dag, hive_table_metadata, fs, context_, prune_level); UInt64 total_rows = 0; for (const auto & hive_file : hive_files) diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 755d71df531..79d87a6e3cb 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool parallelize_output_from_storages; +} + namespace ErrorCodes { extern const int TABLE_IS_DROPPED; @@ -157,7 +162,7 @@ void IStorage::read( /// parallelize processing if not yet const size_t output_ports = pipe.numOutputPorts(); - const bool parallelize_output = context->getSettingsRef().parallelize_output_from_storages; + const bool parallelize_output = context->getSettingsRef()[Setting::parallelize_output_from_storages]; if (parallelize_output && parallelizeOutputAfterReading(context) && output_ports > 0 && output_ports < num_streams) pipe.resize(num_streams); diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 63467603d16..32c3ee4130e 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -29,6 +29,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool async_query_sending_for_remote; + extern const SettingsBool async_socket_for_remote; + extern const SettingsBool skip_unavailable_shards; +} IStorageCluster::IStorageCluster( const String & cluster_name_, @@ -125,7 +132,7 @@ void IStorageCluster::read( Block sample_block; ASTPtr query_to_send = query_info.query; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query_info.query, context, SelectQueryOptions(processed_stage)); } @@ -196,8 +203,8 @@ void ReadFromCluster::initializePipeline(QueryPipelineBuilder & pipeline, const pipes.emplace_back(std::make_shared( remote_query_executor, add_agg_info, - current_settings.async_socket_for_remote, - current_settings.async_query_sending_for_remote)); + current_settings[Setting::async_socket_for_remote], + current_settings[Setting::async_query_sending_for_remote])); } } @@ -225,10 +232,10 @@ QueryProcessingStage::Enum IStorageCluster::getQueryProcessingStage( ContextPtr ReadFromCluster::updateSettings(const Settings & settings) { - Settings new_settings = settings; + Settings new_settings{settings}; /// Cluster table functions should always skip unavailable shards. - new_settings.skip_unavailable_shards = true; + new_settings[Setting::skip_unavailable_shards] = true; auto new_context = Context::createCopy(context); new_context->setSettings(new_settings); diff --git a/src/Storages/Kafka/KafkaSettings.h b/src/Storages/Kafka/KafkaSettings.h index 9ca5e189f0e..6cf881634ad 100644 --- a/src/Storages/Kafka/KafkaSettings.h +++ b/src/Storages/Kafka/KafkaSettings.h @@ -1,7 +1,9 @@ #pragma once #include -#include +#include +#include +#include namespace DB diff --git a/src/Storages/Kafka/KafkaSource.cpp b/src/Storages/Kafka/KafkaSource.cpp index 3ddd0d1be8c..a2c87237af6 100644 --- a/src/Storages/Kafka/KafkaSource.cpp +++ b/src/Storages/Kafka/KafkaSource.cpp @@ -1,11 +1,12 @@ #include +#include #include #include -#include -#include -#include #include +#include +#include +#include #include @@ -19,6 +20,11 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsMilliseconds kafka_max_wait_ms; +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -78,7 +84,7 @@ Chunk KafkaSource::generateImpl() { if (!consumer) { - auto timeout = std::chrono::milliseconds(context->getSettingsRef().kafka_max_wait_ms.totalMilliseconds()); + auto timeout = std::chrono::milliseconds(context->getSettingsRef()[Setting::kafka_max_wait_ms].totalMilliseconds()); consumer = storage.popConsumer(timeout); if (!consumer) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index f4f641d1c68..b5eb8d07f21 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include namespace CurrentMetrics @@ -65,6 +66,15 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsUInt64 output_format_avro_rows_in_file; + extern const SettingsMilliseconds stream_flush_interval_ms; + extern const SettingsMilliseconds stream_poll_timeout_ms; + extern const SettingsBool use_concurrency_control; +} namespace ErrorCodes { @@ -229,7 +239,7 @@ SinkToStoragePtr StorageKafka::write(const ASTPtr &, const StorageMetadataPtr & cppkafka::Configuration conf = getProducerConfiguration(); const Settings & settings = getContext()->getSettingsRef(); - size_t poll_timeout = settings.stream_poll_timeout_ms.totalMilliseconds(); + size_t poll_timeout = settings[Setting::stream_poll_timeout_ms].totalMilliseconds(); const auto & header = metadata_snapshot->getSampleBlockNonMaterialized(); auto producer = std::make_unique( @@ -239,8 +249,8 @@ SinkToStoragePtr StorageKafka::write(const ASTPtr &, const StorageMetadataPtr & size_t max_rows = max_rows_per_message; /// Need for backward compatibility. - if (format_name == "Avro" && local_context->getSettingsRef().output_format_avro_rows_in_file.changed) - max_rows = local_context->getSettingsRef().output_format_avro_rows_in_file.value; + if (format_name == "Avro" && local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].changed) + max_rows = local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].value; return std::make_shared( header, getFormatName(), max_rows, std::move(producer), getName(), modified_context); } @@ -476,25 +486,22 @@ void StorageKafka::cleanConsumers() size_t StorageKafka::getMaxBlockSize() const { - return kafka_settings->kafka_max_block_size.changed - ? kafka_settings->kafka_max_block_size.value - : (getContext()->getSettingsRef().max_insert_block_size.value / num_consumers); + return kafka_settings->kafka_max_block_size.changed ? kafka_settings->kafka_max_block_size.value + : (getContext()->getSettingsRef()[Setting::max_insert_block_size].value / num_consumers); } size_t StorageKafka::getPollMaxBatchSize() const { - size_t batch_size = kafka_settings->kafka_poll_max_batch_size.changed - ? kafka_settings->kafka_poll_max_batch_size.value - : getContext()->getSettingsRef().max_block_size.value; + size_t batch_size = kafka_settings->kafka_poll_max_batch_size.changed ? kafka_settings->kafka_poll_max_batch_size.value + : getContext()->getSettingsRef()[Setting::max_block_size].value; return std::min(batch_size,getMaxBlockSize()); } size_t StorageKafka::getPollTimeoutMillisecond() const { - return kafka_settings->kafka_poll_timeout_ms.changed - ? kafka_settings->kafka_poll_timeout_ms.totalMilliseconds() - : getContext()->getSettingsRef().stream_poll_timeout_ms.totalMilliseconds(); + return kafka_settings->kafka_poll_timeout_ms.changed ? kafka_settings->kafka_poll_timeout_ms.totalMilliseconds() + : getContext()->getSettingsRef()[Setting::stream_poll_timeout_ms].totalMilliseconds(); } void StorageKafka::threadFunc(size_t idx) @@ -619,8 +626,8 @@ bool StorageKafka::streamToViews() StreamLocalLimits limits; Poco::Timespan max_execution_time = kafka_settings->kafka_flush_interval_ms.changed - ? kafka_settings->kafka_flush_interval_ms - : getContext()->getSettingsRef().stream_flush_interval_ms; + ? kafka_settings->kafka_flush_interval_ms + : getContext()->getSettingsRef()[Setting::stream_flush_interval_ms]; source->setTimeLimit(max_execution_time); } @@ -637,7 +644,7 @@ bool StorageKafka::streamToViews() // we need to read all consumers in parallel (sequential read may lead to situation // when some of consumers are not used, and will break some Kafka consumer invariants) block_io.pipeline.setNumThreads(stream_count); - block_io.pipeline.setConcurrencyControl(kafka_context->getSettingsRef().use_concurrency_control); + block_io.pipeline.setConcurrencyControl(kafka_context->getSettingsRef()[Setting::use_concurrency_control]); block_io.pipeline.setProgressCallback([&](const Progress & progress) { rows += progress.read_rows.load(); }); CompletedPipelineExecutor executor(block_io.pipeline); diff --git a/src/Storages/Kafka/StorageKafka2.cpp b/src/Storages/Kafka/StorageKafka2.cpp index 3574b46e3b0..6d167b0bb32 100644 --- a/src/Storages/Kafka/StorageKafka2.cpp +++ b/src/Storages/Kafka/StorageKafka2.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -71,6 +72,14 @@ extern const Event KafkaWrites; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_block_size; + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsUInt64 output_format_avro_rows_in_file; + extern const SettingsMilliseconds stream_flush_interval_ms; + extern const SettingsMilliseconds stream_poll_timeout_ms; +} namespace fs = std::filesystem; @@ -349,7 +358,7 @@ StorageKafka2::write(const ASTPtr &, const StorageMetadataPtr & metadata_snapsho cppkafka::Configuration conf = getProducerConfiguration(); const Settings & settings = getContext()->getSettingsRef(); - size_t poll_timeout = settings.stream_poll_timeout_ms.totalMilliseconds(); + size_t poll_timeout = settings[Setting::stream_poll_timeout_ms].totalMilliseconds(); const auto & header = metadata_snapshot->getSampleBlockNonMaterialized(); auto producer = std::make_unique( @@ -359,8 +368,8 @@ StorageKafka2::write(const ASTPtr &, const StorageMetadataPtr & metadata_snapsho size_t max_rows = max_rows_per_message; /// Need for backward compatibility. - if (format_name == "Avro" && local_context->getSettingsRef().output_format_avro_rows_in_file.changed) - max_rows = local_context->getSettingsRef().output_format_avro_rows_in_file.value; + if (format_name == "Avro" && local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].changed) + max_rows = local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].value; return std::make_shared(header, getFormatName(), max_rows, std::move(producer), getName(), modified_context); } @@ -443,13 +452,13 @@ cppkafka::Configuration StorageKafka2::getProducerConfiguration() size_t StorageKafka2::getMaxBlockSize() const { return kafka_settings->kafka_max_block_size.changed ? kafka_settings->kafka_max_block_size.value - : (getContext()->getSettingsRef().max_insert_block_size.value / num_consumers); + : (getContext()->getSettingsRef()[Setting::max_insert_block_size].value / num_consumers); } size_t StorageKafka2::getPollMaxBatchSize() const { size_t batch_size = kafka_settings->kafka_poll_max_batch_size.changed ? kafka_settings->kafka_poll_max_batch_size.value - : getContext()->getSettingsRef().max_block_size.value; + : getContext()->getSettingsRef()[Setting::max_block_size].value; return std::min(batch_size, getMaxBlockSize()); } @@ -457,7 +466,7 @@ size_t StorageKafka2::getPollMaxBatchSize() const size_t StorageKafka2::getPollTimeoutMillisecond() const { return kafka_settings->kafka_poll_timeout_ms.changed ? kafka_settings->kafka_poll_timeout_ms.totalMilliseconds() - : getContext()->getSettingsRef().stream_poll_timeout_ms.totalMilliseconds(); + : getContext()->getSettingsRef()[Setting::stream_poll_timeout_ms].totalMilliseconds(); } namespace @@ -854,7 +863,7 @@ StorageKafka2::PolledBatchInfo StorageKafka2::pollConsumer( Poco::Timespan max_execution_time = kafka_settings->kafka_flush_interval_ms.changed ? kafka_settings->kafka_flush_interval_ms - : getContext()->getSettingsRef().stream_flush_interval_ms; + : getContext()->getSettingsRef()[Setting::stream_flush_interval_ms]; const auto check_time_limit = [&max_execution_time, &total_stopwatch]() { diff --git a/src/Storages/Kafka/StorageKafkaUtils.cpp b/src/Storages/Kafka/StorageKafkaUtils.cpp index cdc32d775eb..e49626c986b 100644 --- a/src/Storages/Kafka/StorageKafkaUtils.cpp +++ b/src/Storages/Kafka/StorageKafkaUtils.cpp @@ -1,6 +1,14 @@ #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -10,13 +18,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include #include #include #include @@ -46,6 +47,11 @@ extern const Event KafkaConsumerErrors; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_kafka_offsets_storage_in_keeper; + extern const SettingsBool kafka_disable_num_consumers_limit; +} using namespace std::chrono_literals; @@ -164,7 +170,7 @@ void registerStorageKafka(StorageFactory & factory) auto num_consumers = kafka_settings->kafka_num_consumers.value; auto max_consumers = std::max(getNumberOfPhysicalCPUCores(), 16); - if (!args.getLocalContext()->getSettingsRef().kafka_disable_num_consumers_limit && num_consumers > max_consumers) + if (!args.getLocalContext()->getSettingsRef()[Setting::kafka_disable_num_consumers_limit] && num_consumers > max_consumers) { throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -217,7 +223,7 @@ void registerStorageKafka(StorageFactory & factory) return std::make_shared( args.table_id, args.getContext(), args.columns, args.comment, std::move(kafka_settings), collection_name); - if (!args.getLocalContext()->getSettingsRef().allow_experimental_kafka_offsets_storage_in_keeper && !args.query.attach) + if (!args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_kafka_offsets_storage_in_keeper] && !args.query.attach) throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, "Storing the Kafka offsets in Keeper is experimental. Set `allow_experimental_kafka_offsets_storage_in_keeper` setting " diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 71b1a0a73c9..df61df6e586 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -55,6 +55,17 @@ limitations under the License. */ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_live_view; + extern const SettingsSeconds live_view_heartbeat_interval; + extern const SettingsUInt64 max_live_view_insert_blocks_before_refresh; + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_rows; + extern const SettingsBool use_concurrency_control; +} + namespace ErrorCodes { extern const int INCORRECT_QUERY; @@ -313,13 +324,21 @@ Pipe StorageLiveView::watch( if (query.is_watch_events) reader = Pipe(std::make_shared( std::static_pointer_cast(shared_from_this()), - blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, - local_context->getSettingsRef().live_view_heartbeat_interval.totalSeconds())); + blocks_ptr, + blocks_metadata_ptr, + active_ptr, + has_limit, + limit, + local_context->getSettingsRef()[Setting::live_view_heartbeat_interval].totalSeconds())); else reader = Pipe(std::make_shared( std::static_pointer_cast(shared_from_this()), - blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit, - local_context->getSettingsRef().live_view_heartbeat_interval.totalSeconds())); + blocks_ptr, + blocks_metadata_ptr, + active_ptr, + has_limit, + limit, + local_context->getSettingsRef()[Setting::live_view_heartbeat_interval].totalSeconds())); { std::lock_guard lock(mutex); @@ -354,7 +373,9 @@ void StorageLiveView::writeBlock(StorageLiveView & live_view, Block && block, Ch { std::lock_guard lock(mutex); - if (!mergeable_blocks || mergeable_blocks->blocks->size() >= local_context->getGlobalContext()->getSettingsRef().max_live_view_insert_blocks_before_refresh) + if (!mergeable_blocks + || mergeable_blocks->blocks->size() + >= local_context->getGlobalContext()->getSettingsRef()[Setting::max_live_view_insert_blocks_before_refresh]) { mergeable_blocks = collectMergeableBlocks(local_context, lock); from = blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block); @@ -378,7 +399,7 @@ void StorageLiveView::writeBlock(StorageLiveView & live_view, Block && block, Ch QueryPipelineBuilder builder; - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { auto select_description = buildSelectQueryTreeDescription(select_query_description.inner_query, local_context); if (select_description.dependent_table_node) @@ -454,7 +475,7 @@ void StorageLiveView::writeBlock(StorageLiveView & live_view, Block && block, Ch }); auto executor = pipeline.execute(); - executor->execute(pipeline.getNumThreads(), local_context->getSettingsRef().use_concurrency_control); + executor->execute(pipeline.getNumThreads(), local_context->getSettingsRef()[Setting::use_concurrency_control]); } void StorageLiveView::refresh() @@ -475,7 +496,7 @@ Block StorageLiveView::getHeader() const if (!sample_block) { - if (live_view_context->getSettingsRef().allow_experimental_analyzer) + if (live_view_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(select_query_description.select_query, live_view_context, @@ -519,7 +540,7 @@ ASTPtr StorageLiveView::getInnerBlocksQuery() auto & select_with_union_query = select_query_description.select_query->as(); auto blocks_query = select_with_union_query.list_of_selects->children.at(0)->clone(); - if (!live_view_context->getSettingsRef().allow_experimental_analyzer) + if (!live_view_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { /// Rewrite inner query with right aliases for JOIN. /// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table, @@ -543,7 +564,7 @@ MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_cont QueryPipelineBuilder builder; - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(select_query_description.inner_query, local_context, @@ -599,7 +620,7 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes) QueryPipelineBuilder builder; - if (block_context->getSettingsRef().allow_experimental_analyzer) + if (block_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { auto select_description = buildSelectQueryTreeDescription(select_query_description.select_query, block_context); @@ -641,13 +662,14 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes) /// Squashing is needed here because the view query can generate a lot of blocks /// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY /// and two-level aggregation is triggered). - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared( - cur_header, - getContext()->getSettingsRef().min_insert_block_size_rows, - getContext()->getSettingsRef().min_insert_block_size_bytes); - }); + builder.addSimpleTransform( + [&](const Block & cur_header) + { + return std::make_shared( + cur_header, + getContext()->getSettingsRef()[Setting::min_insert_block_size_rows], + getContext()->getSettingsRef()[Setting::min_insert_block_size_bytes]); + }); builder.addResources(std::move(resource_holder)); @@ -724,14 +746,17 @@ bool StorageLiveView::getNewBlocks(const std::lock_guard & lock) void registerStorageLiveView(StorageFactory & factory) { - factory.registerStorage("LiveView", [](const StorageFactory::Arguments & args) - { - if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef().allow_experimental_live_view) - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')"); + factory.registerStorage( + "LiveView", + [](const StorageFactory::Arguments & args) + { + if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_live_view]) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')"); - return std::make_shared(args.table_id, args.getLocalContext(), args.query, args.columns, args.comment); - }); + return std::make_shared(args.table_id, args.getLocalContext(), args.query, args.columns, args.comment); + }); } } diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index ed5a6652288..93cab24af7f 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -18,6 +18,11 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 log_queries_cut_to_length; + extern const SettingsBool stop_refreshable_materialized_views_on_startup; +} namespace ErrorCodes { @@ -56,7 +61,7 @@ OwnedRefreshTask RefreshTask::create( void RefreshTask::initializeAndStart() { - if (view->getContext()->getSettingsRef().stop_refreshable_materialized_views_on_startup) + if (view->getContext()->getSettingsRef()[Setting::stop_refreshable_materialized_views_on_startup]) stop_requested = true; view->getContext()->getRefreshSet().emplace(view->getStorageID(), initial_dependencies, shared_from_this()); populateDependencies(); @@ -399,8 +404,7 @@ void RefreshTask::executeRefreshUnlocked(bool append) }); /// Add the query to system.processes and allow it to be killed with KILL QUERY. - String query_for_logging = refresh_query->formatForLogging( - refresh_context->getSettingsRef().log_queries_cut_to_length); + String query_for_logging = refresh_query->formatForLogging(refresh_context->getSettingsRef()[Setting::log_queries_cut_to_length]); block_io.process_list_entry = refresh_context->getProcessList().insert( query_for_logging, refresh_query.get(), refresh_context, Stopwatch{CLOCK_MONOTONIC}.getStart()); pipeline.setProcessListElement(block_io.process_list_entry->getQueryStatus()); diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index adaa4eddb98..2fdb0794789 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 1ed096fae17..dbfc1074ef7 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -42,6 +42,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool analyze_index_with_space_filling_curves; +} namespace ErrorCodes { @@ -818,7 +822,7 @@ KeyCondition::KeyCondition( ++key_index; } - if (context->getSettingsRef().analyze_index_with_space_filling_curves) + if (context->getSettingsRef()[Setting::analyze_index_with_space_filling_curves]) getAllSpaceFillingCurves(); if (!filter_dag) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 0beeffcb267..4dd8400ec96 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -62,6 +62,14 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool compile_sort_description; + extern const SettingsUInt64 max_insert_delayed_streams_for_parallel_write; + extern const SettingsUInt64 min_count_to_compile_sort_description; + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_rows; +} namespace ErrorCodes { @@ -656,7 +664,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::prepareProjectionsToMergeAndRe for (const auto * projection : global_ctx->projections_to_rebuild) ctx->projection_squashes.emplace_back(projection->sample_block.cloneEmpty(), - settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes); + settings[Setting::min_insert_block_size_rows], settings[Setting::min_insert_block_size_bytes]); } @@ -837,8 +845,8 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const size_t max_delayed_streams = 0; if (global_ctx->new_data_part->getDataPartStorage().supportParallelWrite()) { - if (settings.max_insert_delayed_streams_for_parallel_write.changed) - max_delayed_streams = settings.max_insert_delayed_streams_for_parallel_write; + if (settings[Setting::max_insert_delayed_streams_for_parallel_write].changed) + max_delayed_streams = settings[Setting::max_insert_delayed_streams_for_parallel_write]; else max_delayed_streams = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE; } @@ -1635,8 +1643,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const { Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); SortDescription sort_description; - sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; - sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef().min_count_to_compile_sort_description; + sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef()[Setting::compile_sort_description]; + sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef()[Setting::min_count_to_compile_sort_description]; size_t sort_columns_size = sort_columns.size(); sort_description.reserve(sort_columns_size); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 80d61058d08..c270c262270 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -153,6 +153,30 @@ namespace namespace DB { +namespace Setting +{ + extern const SettingsBool allow_drop_detached; + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_full_text_index; + extern const SettingsBool allow_experimental_inverted_index; + extern const SettingsBool allow_experimental_vector_similarity_index; + extern const SettingsBool allow_non_metadata_alters; + extern const SettingsBool allow_statistics_optimize; + extern const SettingsBool allow_suspicious_indices; + extern const SettingsBool alter_move_to_space_execute_async; + extern const SettingsBool alter_partition_verbose_result; + extern const SettingsBool apply_mutations_on_fly; + extern const SettingsBool fsync_metadata; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool materialize_ttl_after_modify; + extern const SettingsUInt64 max_partition_size_to_drop; + extern const SettingsMaxThreads max_threads; + extern const SettingsUInt64 number_of_mutations_to_delay; + extern const SettingsUInt64 number_of_mutations_to_throw; + extern const SettingsBool parallel_replicas_for_non_replicated_merge_tree; + extern const SettingsUInt64 parts_to_delay_insert; + extern const SettingsUInt64 parts_to_throw_insert; +} namespace ErrorCodes { @@ -306,7 +330,7 @@ void MergeTreeData::initializeDirectoriesAndFormatVersion(const std::string & re auto buf = disk->writeFile(format_version_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, getContext()->getWriteSettings()); writeIntText(format_version.toUnderType(), *buf); buf->finalize(); - if (getContext()->getSettingsRef().fsync_metadata) + if (getContext()->getSettingsRef()[Setting::fsync_metadata]) buf->sync(); } @@ -476,7 +500,7 @@ StoragePolicyPtr MergeTreeData::getStoragePolicy() const ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByPredicate( const StorageSnapshotPtr & storage_snapshot, const ActionsDAG * filter_dag, ContextPtr local_context) const { - if (!local_context->getSettingsRef().allow_statistics_optimize) + if (!local_context->getSettingsRef()[Setting::allow_statistics_optimize]) return {}; const auto & parts = assert_cast(*storage_snapshot->data).parts; @@ -591,7 +615,7 @@ void MergeTreeData::checkProperties( bool allow_suspicious_indices = getSettings()->allow_suspicious_indices; if (local_context) - allow_suspicious_indices = local_context->getSettingsRef().allow_suspicious_indices; + allow_suspicious_indices = local_context->getSettingsRef()[Setting::allow_suspicious_indices]; if (!allow_suspicious_indices && !attach) if (const auto * index_function = typeid_cast(new_sorting_key.definition_ast.get())) @@ -3202,9 +3226,9 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context const auto & settings = local_context->getSettingsRef(); const auto & settings_from_storage = getSettings(); - if (!settings.allow_non_metadata_alters) + if (!settings[Setting::allow_non_metadata_alters]) { - auto mutation_commands = commands.getMutationCommands(new_metadata, settings.materialize_ttl_after_modify, local_context); + auto mutation_commands = commands.getMutationCommands(new_metadata, settings[Setting::materialize_ttl_after_modify], local_context); if (!mutation_commands.empty()) throw Exception(ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN, @@ -3226,15 +3250,15 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context commands.apply(new_metadata, local_context); - if (AlterCommands::hasFullTextIndex(new_metadata) && !settings.allow_experimental_full_text_index) + if (AlterCommands::hasFullTextIndex(new_metadata) && !settings[Setting::allow_experimental_full_text_index]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental full-text index feature is not enabled (turn on setting 'allow_experimental_full_text_index')"); - if (AlterCommands::hasLegacyInvertedIndex(new_metadata) && !settings.allow_experimental_inverted_index) + if (AlterCommands::hasLegacyInvertedIndex(new_metadata) && !settings[Setting::allow_experimental_inverted_index]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental inverted index feature is not enabled (turn on setting 'allow_experimental_inverted_index')"); - if (AlterCommands::hasVectorSimilarityIndex(new_metadata) && !settings.allow_experimental_vector_similarity_index) + if (AlterCommands::hasVectorSimilarityIndex(new_metadata) && !settings[Setting::allow_experimental_vector_similarity_index]) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental vector similarity index is disabled (turn on setting 'allow_experimental_vector_similarity_index')"); @@ -4566,9 +4590,9 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, const Contex auto [parts_count_in_partition, size_of_partition] = getMaxPartsCountAndSizeForPartition(); size_t average_part_size = parts_count_in_partition ? size_of_partition / parts_count_in_partition : 0; const auto active_parts_to_delay_insert - = query_settings.parts_to_delay_insert ? query_settings.parts_to_delay_insert : settings->parts_to_delay_insert; + = query_settings[Setting::parts_to_delay_insert] ? query_settings[Setting::parts_to_delay_insert] : settings->parts_to_delay_insert; const auto active_parts_to_throw_insert - = query_settings.parts_to_throw_insert ? query_settings.parts_to_throw_insert : settings->parts_to_throw_insert; + = query_settings[Setting::parts_to_throw_insert] ? query_settings[Setting::parts_to_throw_insert] : settings->parts_to_throw_insert; size_t active_parts_over_threshold = 0; { @@ -4645,13 +4669,11 @@ void MergeTreeData::delayMutationOrThrowIfNeeded(Poco::Event * until, const Cont const auto settings = getSettings(); const auto & query_settings = query_context->getSettingsRef(); - size_t num_mutations_to_delay = query_settings.number_of_mutations_to_delay - ? query_settings.number_of_mutations_to_delay - : settings->number_of_mutations_to_delay; + size_t num_mutations_to_delay = query_settings[Setting::number_of_mutations_to_delay] ? query_settings[Setting::number_of_mutations_to_delay] + : settings->number_of_mutations_to_delay; - size_t num_mutations_to_throw = query_settings.number_of_mutations_to_throw - ? query_settings.number_of_mutations_to_throw - : settings->number_of_mutations_to_throw; + size_t num_mutations_to_throw = query_settings[Setting::number_of_mutations_to_throw] ? query_settings[Setting::number_of_mutations_to_throw] + : settings->number_of_mutations_to_throw; if (!num_mutations_to_delay && !num_mutations_to_throw) return; @@ -4997,8 +5019,7 @@ void MergeTreeData::checkAlterPartitionIsPossible( for (const auto & command : commands) { - if (command.type == PartitionCommand::DROP_DETACHED_PARTITION - && !settings.allow_drop_detached) + if (command.type == PartitionCommand::DROP_DETACHED_PARTITION && !settings[Setting::allow_drop_detached]) throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Cannot execute query: DROP DETACHED PART " "is disabled (see allow_drop_detached setting)"); @@ -5058,9 +5079,10 @@ void MergeTreeData::checkPartitionCanBeDropped(const ASTPtr & partition, Context auto table_id = getStorageID(); const auto & query_settings = local_context->getSettingsRef(); - if (query_settings.max_partition_size_to_drop.changed) + if (query_settings[Setting::max_partition_size_to_drop].changed) { - getContext()->checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, partition_size, query_settings.max_partition_size_to_drop); + getContext()->checkPartitionCanBeDropped( + table_id.database_name, table_id.table_name, partition_size, query_settings[Setting::max_partition_size_to_drop]); return; } @@ -5079,9 +5101,10 @@ void MergeTreeData::checkPartCanBeDropped(const String & part_name, ContextPtr l auto table_id = getStorageID(); const auto & query_settings = local_context->getSettingsRef(); - if (query_settings.max_partition_size_to_drop.changed) + if (query_settings[Setting::max_partition_size_to_drop].changed) { - getContext()->checkPartitionCanBeDropped(table_id.database_name, table_id.table_name, part->getBytesOnDisk(), query_settings.max_partition_size_to_drop); + getContext()->checkPartitionCanBeDropped( + table_id.database_name, table_id.table_name, part->getBytesOnDisk(), query_settings[Setting::max_partition_size_to_drop]); return; } @@ -5130,9 +5153,13 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No parts to move are found in partition {}", partition_id); const auto & query_settings = local_context->getSettingsRef(); - std::future moves_future = movePartsToSpace(moving_tagger, local_context->getReadSettings(), local_context->getWriteSettings(), query_settings.alter_move_to_space_execute_async); + std::future moves_future = movePartsToSpace( + moving_tagger, + local_context->getReadSettings(), + local_context->getWriteSettings(), + query_settings[Setting::alter_move_to_space_execute_async]); - if (query_settings.alter_move_to_space_execute_async && moves_future.wait_for(std::chrono::seconds(0)) != std::future_status::ready) + if (query_settings[Setting::alter_move_to_space_execute_async] && moves_future.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { return; } @@ -5211,9 +5238,13 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No parts to move are found in partition {}", partition_id); const auto & query_settings = local_context->getSettingsRef(); - std::future moves_future = movePartsToSpace(moving_tagger, local_context->getReadSettings(), local_context->getWriteSettings(), query_settings.alter_move_to_space_execute_async); + std::future moves_future = movePartsToSpace( + moving_tagger, + local_context->getReadSettings(), + local_context->getWriteSettings(), + query_settings[Setting::alter_move_to_space_execute_async]); - if (query_settings.alter_move_to_space_execute_async && moves_future.wait_for(std::chrono::seconds(0)) != std::future_status::ready) + if (query_settings[Setting::alter_move_to_space_execute_async] && moves_future.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { return; } @@ -5372,28 +5403,28 @@ Pipe MergeTreeData::alterPartition( case PartitionCommand::FREEZE_PARTITION: { - auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); current_command_results = freezePartition(command.partition, command.with_name, query_context, lock); } break; case PartitionCommand::FREEZE_ALL_PARTITIONS: { - auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); current_command_results = freezeAll(command.with_name, query_context, lock); } break; case PartitionCommand::UNFREEZE_PARTITION: { - auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); current_command_results = unfreezePartition(command.partition, command.with_name, query_context, lock); } break; case PartitionCommand::UNFREEZE_ALL_PARTITIONS: { - auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); current_command_results = unfreezeAll(command.with_name, query_context, lock); } @@ -5407,7 +5438,7 @@ Pipe MergeTreeData::alterPartition( result.insert(result.end(), current_command_results.begin(), current_command_results.end()); } - if (query_context->getSettingsRef().alter_partition_verbose_result) + if (query_context->getSettingsRef()[Setting::alter_partition_verbose_result]) return convertCommandsResultToSource(result); return {}; @@ -5448,7 +5479,7 @@ MergeTreeData::PartsBackupEntries MergeTreeData::backupParts( } if (hold_table_lock && !table_lock) - table_lock = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + table_lock = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); if (backup_settings.check_projection_parts) part->checkConsistencyWithProjections(/* require_part_metadata= */ true); @@ -7124,7 +7155,7 @@ QueryProcessingStage::Enum MergeTreeData::getQueryProcessingStage( SelectQueryInfo &) const { /// with new analyzer, Planner make decision regarding parallel replicas usage, and so about processing stage on reading - if (!query_context->getSettingsRef().allow_experimental_analyzer) + if (!query_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { const auto & settings = query_context->getSettingsRef(); if (query_context->canUseParallelReplicasCustomKey()) @@ -7132,7 +7163,7 @@ QueryProcessingStage::Enum MergeTreeData::getQueryProcessingStage( if (query_context->getClientInfo().distributed_depth > 0) return QueryProcessingStage::FetchColumns; - if (!supportsReplication() && !settings.parallel_replicas_for_non_replicated_merge_tree) + if (!supportsReplication() && !settings[Setting::parallel_replicas_for_non_replicated_merge_tree]) return QueryProcessingStage::Enum::FetchColumns; if (to_stage >= QueryProcessingStage::WithMergeableState @@ -7151,7 +7182,7 @@ QueryProcessingStage::Enum MergeTreeData::getQueryProcessingStage( return QueryProcessingStage::Enum::WithMergeableState; /// For non-replicated MergeTree we allow them only if parallel_replicas_for_non_replicated_merge_tree is enabled - if (settings.parallel_replicas_for_non_replicated_merge_tree) + if (settings[Setting::parallel_replicas_for_non_replicated_merge_tree]) return QueryProcessingStage::Enum::WithMergeableState; } } @@ -7173,7 +7204,7 @@ UInt64 MergeTreeData::estimateNumberOfRowsToRead( storage_snapshot->metadata, query_info, query_context, - query_context->getSettingsRef().max_threads); + query_context->getSettingsRef()[Setting::max_threads]); UInt64 total_rows = result_ptr->selected_rows; if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows) @@ -8493,7 +8524,7 @@ bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr & return false; if (!storage_snapshot) - return !query_context->getSettingsRef().apply_mutations_on_fly; + return !query_context->getSettingsRef()[Setting::apply_mutations_on_fly]; const auto & snapshot_data = assert_cast(*storage_snapshot->data); return !snapshot_data.mutations_snapshot->hasDataMutations(); @@ -8526,7 +8557,7 @@ StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & { .metadata_version = metadata_snapshot->getMetadataVersion(), .min_part_metadata_version = getMinMetadataVersion(snapshot_data->parts), - .need_data_mutations = query_context->getSettingsRef().apply_mutations_on_fly, + .need_data_mutations = query_context->getSettingsRef()[Setting::apply_mutations_on_fly], }; snapshot_data->mutations_snapshot = getMutationsSnapshot(params); @@ -8690,7 +8721,7 @@ bool MergeTreeData::initializeDiskOnConfigChange(const std::set & new_ad auto buf = disk->writeFile(format_version_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, getContext()->getWriteSettings()); writeIntText(format_version.toUnderType(), *buf); buf->finalize(); - if (getContext()->getSettingsRef().fsync_metadata) + if (getContext()->getSettingsRef()[Setting::fsync_metadata]) buf->sync(); } } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 6e0ae8f7cca..4f75746d6c2 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -64,6 +64,23 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_query_deduplication; + extern const SettingsString force_data_skipping_indices; + extern const SettingsBool force_index_by_date; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsInt64 max_partitions_to_read; + extern const SettingsUInt64 max_query_size; + extern const SettingsUInt64 max_threads_for_indexes; + extern const SettingsUInt64 merge_tree_coarse_index_granularity; + extern const SettingsUInt64 merge_tree_min_bytes_for_seek; + extern const SettingsUInt64 merge_tree_min_rows_for_seek; + extern const SettingsUInt64 parallel_replica_offset; + extern const SettingsUInt64 parallel_replicas_count; +} namespace ErrorCodes { @@ -280,8 +297,8 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( auto parallel_replicas_mode = context->getParallelReplicasMode(); /// Parallel replicas has been requested but there is no way to sample data. /// Select all data from first replica and no data from other replicas. - if (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY - && !data.supportsSampling() && settings.parallel_replica_offset > 0) + if (settings[Setting::parallel_replicas_count] > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + && !data.supportsSampling() && settings[Setting::parallel_replica_offset] > 0) { LOG_DEBUG( log, @@ -292,7 +309,7 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( } sampling.use_sampling = relative_sample_size > 0 - || (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + || (settings[Setting::parallel_replicas_count] > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY && data.supportsSampling()); bool no_data = false; /// There is nothing left after sampling. @@ -322,13 +339,13 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( "Invalid sampling column type in storage parameters: {}. Must be one unsigned integer type", sampling_column_type->getName()); - if (settings.parallel_replicas_count > 1) + if (settings[Setting::parallel_replicas_count] > 1) { if (relative_sample_size == RelativeSize(0)) relative_sample_size = 1; - relative_sample_size /= settings.parallel_replicas_count.value; - relative_sample_offset += relative_sample_size * RelativeSize(settings.parallel_replica_offset.value); + relative_sample_size /= settings[Setting::parallel_replicas_count].value; + relative_sample_offset += relative_sample_size * RelativeSize(settings[Setting::parallel_replica_offset].value); } if (relative_sample_offset >= RelativeSize(1)) @@ -526,7 +543,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( const auto & partition_key = metadata_snapshot->getPartitionKey(); minmax_columns_types = MergeTreeData::getMinMaxColumnsTypes(partition_key); - if (settings.force_index_by_date && (minmax_idx_condition->alwaysUnknownOrTrue() && partition_pruner->isUseless())) + if (settings[Setting::force_index_by_date] && (minmax_idx_condition->alwaysUnknownOrTrue() && partition_pruner->isUseless())) { auto minmax_columns_names = MergeTreeData::getMinMaxColumnsNames(partition_key); throw Exception(ErrorCodes::INDEX_NOT_USED, @@ -539,7 +556,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( QueryStatusPtr query_status = context->getProcessListElement(); PartFilterCounters part_filter_counters; - if (query_context->getSettingsRef().allow_experimental_query_deduplication) + if (query_context->getSettingsRef()[Setting::allow_experimental_query_deduplication]) selectPartsToReadWithUUIDFilter( parts, part_values, @@ -609,14 +626,15 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd parts_with_ranges.resize(parts.size()); const Settings & settings = context->getSettingsRef(); - if (use_skip_indexes && settings.force_data_skipping_indices.changed) + if (use_skip_indexes && settings[Setting::force_data_skipping_indices].changed) { - const auto & indices = settings.force_data_skipping_indices.toString(); + const auto & indices = settings[Setting::force_data_skipping_indices].toString(); Strings forced_indices; { - Tokens tokens(indices.data(), indices.data() + indices.size(), settings.max_query_size); - IParser::Pos pos(tokens, static_cast(settings.max_parser_depth), static_cast(settings.max_parser_backtracks)); + Tokens tokens(indices.data(), indices.data() + indices.size(), settings[Setting::max_query_size]); + IParser::Pos pos( + tokens, static_cast(settings[Setting::max_parser_depth]), static_cast(settings[Setting::max_parser_backtracks])); Expected expected; if (!parseIdentifiersOrStringLiterals(pos, expected, forced_indices)) throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse force_data_skipping_indices ('{}')", indices); @@ -751,9 +769,9 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd }; size_t num_threads = std::min(num_streams, parts.size()); - if (settings.max_threads_for_indexes) + if (settings[Setting::max_threads_for_indexes]) { - num_threads = std::min(num_streams, settings.max_threads_for_indexes); + num_threads = std::min(num_streams, settings[Setting::max_threads_for_indexes]); } LOG_TRACE(log, "Filtering marks by primary and secondary keys"); @@ -781,19 +799,19 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd /// So we wait until lock_acquire_timeout, and then raise an exception. for (size_t part_index = 0; part_index < parts.size(); ++part_index) { - pool.scheduleOrThrow([&, part_index, thread_group = CurrentThread::getGroup()] - { - setThreadName("MergeTreeIndex"); + pool.scheduleOrThrow( + [&, part_index, thread_group = CurrentThread::getGroup()] + { + setThreadName("MergeTreeIndex"); - SCOPE_EXIT_SAFE( + SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached();); if (thread_group) - CurrentThread::detachFromGroupIfNotDetached(); - ); - if (thread_group) - CurrentThread::attachToGroupIfDetached(thread_group); + CurrentThread::attachToGroupIfDetached(thread_group); - process_part(part_index); - }, Priority{}, context->getSettingsRef().lock_acquire_timeout.totalMicroseconds()); + process_part(part_index); + }, + Priority{}, + context->getSettingsRef()[Setting::lock_acquire_timeout].totalMicroseconds()); } pool.wait(); @@ -881,7 +899,7 @@ std::shared_ptr MergeTreeDataSelectExecutor::checkLimits( const auto & settings = context->getSettingsRef(); const auto data_settings = data.getSettings(); auto max_partitions_to_read - = settings.max_partitions_to_read.changed ? settings.max_partitions_to_read : data_settings->max_partitions_to_read; + = settings[Setting::max_partitions_to_read].changed ? settings[Setting::max_partitions_to_read] : data_settings->max_partitions_to_read; if (max_partitions_to_read > 0) { std::set partitions; @@ -1184,12 +1202,12 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { // Do exclusion search, where we drop ranges that do not match - if (settings.merge_tree_coarse_index_granularity <= 1) + if (settings[Setting::merge_tree_coarse_index_granularity] <= 1) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Setting merge_tree_coarse_index_granularity should be greater than 1"); size_t min_marks_for_seek = roundRowsOrBytesToMarks( - settings.merge_tree_min_rows_for_seek, - settings.merge_tree_min_bytes_for_seek, + settings[Setting::merge_tree_min_rows_for_seek], + settings[Setting::merge_tree_min_bytes_for_seek], part->index_granularity_info.fixed_index_granularity, part->index_granularity_info.index_granularity_bytes); @@ -1232,7 +1250,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( else { /// Break the segment and put the result on the stack from right to left. - size_t step = (range.end - range.begin - 1) / settings.merge_tree_coarse_index_granularity + 1; + size_t step = (range.end - range.begin - 1) / settings[Setting::merge_tree_coarse_index_granularity] + 1; size_t end; for (end = range.end; end > range.begin + step; end -= step) @@ -1366,8 +1384,8 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( auto index_granularity = index_helper->index.granularity; const size_t min_marks_for_seek = roundRowsOrBytesToMarks( - settings.merge_tree_min_rows_for_seek, - settings.merge_tree_min_bytes_for_seek, + settings[Setting::merge_tree_min_rows_for_seek], + settings[Setting::merge_tree_min_bytes_for_seek], part->index_granularity_info.fixed_index_granularity, part->index_granularity_info.index_granularity_bytes); @@ -1484,8 +1502,8 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex( auto index_granularity = indices.front()->index.granularity; const size_t min_marks_for_seek = roundRowsOrBytesToMarks( - settings.merge_tree_min_rows_for_seek, - settings.merge_tree_min_bytes_for_seek, + settings[Setting::merge_tree_min_rows_for_seek], + settings[Setting::merge_tree_min_bytes_for_seek], part->index_granularity_info.fixed_index_granularity, part->index_granularity_info.index_granularity_bytes); diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index f29d715e791..ed68200041b 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -54,6 +54,13 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool materialize_skip_indexes_on_insert; + extern const SettingsBool materialize_statistics_on_insert; + extern const SettingsBool optimize_on_insert; + extern const SettingsBool throw_on_max_partitions_per_insert_block; +} namespace ErrorCodes { @@ -78,7 +85,7 @@ void buildScatterSelector( size_t num_rows = columns[0]->size(); size_t partitions_count = 0; - size_t throw_on_limit = context->getSettingsRef().throw_on_max_partitions_per_insert_block; + size_t throw_on_limit = context->getSettingsRef()[Setting::throw_on_max_partitions_per_insert_block]; for (size_t i = 0; i < num_rows; ++i) { @@ -484,11 +491,11 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( temp_part.temporary_directory_lock = data.getTemporaryPartDirectoryHolder(part_dir); MergeTreeIndices indices; - if (context->getSettingsRef().materialize_skip_indexes_on_insert) + if (context->getSettingsRef()[Setting::materialize_skip_indexes_on_insert]) indices = MergeTreeIndexFactory::instance().getMany(metadata_snapshot->getSecondaryIndices()); ColumnsStatistics statistics; - if (context->getSettingsRef().materialize_statistics_on_insert) + if (context->getSettingsRef()[Setting::materialize_statistics_on_insert]) statistics = MergeTreeStatisticsFactory::instance().getMany(metadata_snapshot->getColumns()); /// If we need to calculate some columns to sort. @@ -529,7 +536,7 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl( } Names partition_key_columns = metadata_snapshot->getPartitionKey().column_names; - if (context->getSettingsRef().optimize_on_insert) + if (context->getSettingsRef()[Setting::optimize_on_insert]) { ProfileEventTimeIncrement watch(ProfileEvents::MergeTreeDataWriterMergingBlocksMicroseconds); block = mergeBlock(std::move(block), sort_description, partition_key_columns, perm_ptr, data.merging_params); diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.cpp b/src/Storages/MergeTree/MergeTreeIOSettings.cpp index 19365a90a14..e953dda4f21 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeIOSettings.cpp @@ -6,6 +6,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 low_cardinality_max_dictionary_size; + extern const SettingsBool low_cardinality_use_single_dictionary_for_part; + extern const SettingsUInt64 min_compress_block_size; + extern const SettingsUInt64 max_compress_block_size; +} MergeTreeWriterSettings::MergeTreeWriterSettings( const Settings & global_settings, @@ -15,9 +22,9 @@ MergeTreeWriterSettings::MergeTreeWriterSettings( bool rewrite_primary_key_, bool blocks_are_granules_size_) : min_compress_block_size( - storage_settings->min_compress_block_size ? storage_settings->min_compress_block_size : global_settings.min_compress_block_size) + storage_settings->min_compress_block_size ? storage_settings->min_compress_block_size : global_settings[Setting::min_compress_block_size]) , max_compress_block_size( - storage_settings->max_compress_block_size ? storage_settings->max_compress_block_size : global_settings.max_compress_block_size) + storage_settings->max_compress_block_size ? storage_settings->max_compress_block_size : global_settings[Setting::max_compress_block_size]) , marks_compression_codec(storage_settings->marks_compression_codec) , marks_compress_block_size(storage_settings->marks_compress_block_size) , compress_primary_key(storage_settings->compress_primary_key) @@ -27,8 +34,8 @@ MergeTreeWriterSettings::MergeTreeWriterSettings( , rewrite_primary_key(rewrite_primary_key_) , blocks_are_granules_size(blocks_are_granules_size_) , query_write_settings(query_write_settings_) - , low_cardinality_max_dictionary_size(global_settings.low_cardinality_max_dictionary_size) - , low_cardinality_use_single_dictionary_for_part(global_settings.low_cardinality_use_single_dictionary_for_part != 0) + , low_cardinality_max_dictionary_size(global_settings[Setting::low_cardinality_max_dictionary_size]) + , low_cardinality_use_single_dictionary_for_part(global_settings[Setting::low_cardinality_use_single_dictionary_for_part] != 0) , use_compact_variant_discriminators_serialization(storage_settings->use_compact_variant_discriminators_serialization) , use_adaptive_write_buffer_for_dynamic_subcolumns(storage_settings->use_adaptive_write_buffer_for_dynamic_subcolumns) , adaptive_write_buffer_initial_size(storage_settings->adaptive_write_buffer_initial_size) diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h index c52d7ffe131..55edc682a1b 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.h +++ b/src/Storages/MergeTree/MergeTreeIndices.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -22,6 +21,8 @@ constexpr auto INDEX_FILE_PREFIX = "skp_idx_"; namespace DB { +struct MergeTreeWriterSettings; + namespace ErrorCodes { extern const int NOT_IMPLEMENTED; diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp index 32e84dbcdea..a99172c4acd 100644 --- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp @@ -25,6 +25,15 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 filesystem_prefetches_limit; + extern const SettingsUInt64 filesystem_prefetch_max_memory_usage; + extern const SettingsUInt64 filesystem_prefetch_step_marks; + extern const SettingsUInt64 filesystem_prefetch_step_bytes; + extern const SettingsBool merge_tree_determine_task_size_by_prewhere_columns; + extern const SettingsUInt64 prefetch_buffer_size; +} namespace ErrorCodes { @@ -329,7 +338,7 @@ void MergeTreePrefetchedReadPool::fillPerPartStatistics() for (const auto & range : parts_ranges[i].ranges) part_stat.sum_marks += range.end - range.begin; - const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info + const auto & columns = settings[Setting::merge_tree_determine_task_size_by_prewhere_columns] && prewhere_info ? prewhere_info->prewhere_actions.getRequiredColumnsNames() : column_names; @@ -338,13 +347,13 @@ void MergeTreePrefetchedReadPool::fillPerPartStatistics() auto update_stat_for_column = [&](const auto & column_name) { size_t column_size = read_info.data_part->getColumnSize(column_name).data_compressed; - part_stat.estimated_memory_usage_for_single_prefetch += std::min(column_size, settings.prefetch_buffer_size); + part_stat.estimated_memory_usage_for_single_prefetch += std::min(column_size, settings[Setting::prefetch_buffer_size]); ++part_stat.required_readers_num; }; /// adjustBufferSize(), which is done in MergeTreeReaderStream and MergeTreeReaderCompact, /// lowers buffer size if file size (or required read range) is less. So we know that the - /// settings.prefetch_buffer_size will be lowered there, therefore we account it here as well. + /// settings[Setting::prefetch_buffer_size] will be lowered there, therefore we account it here as well. /// But here we make a more approximate lowering (because we do not have loaded marks yet), /// while in adjustBufferSize it will be presize. for (const auto & column : read_info.task_columns.columns) @@ -384,14 +393,16 @@ void MergeTreePrefetchedReadPool::fillPerThreadTasks(size_t threads, size_t sum_ { auto & part_stat = per_part_statistics[i]; - if (settings.filesystem_prefetch_step_marks) + if (settings[Setting::filesystem_prefetch_step_marks]) { - part_stat.prefetch_step_marks = settings.filesystem_prefetch_step_marks; + part_stat.prefetch_step_marks = settings[Setting::filesystem_prefetch_step_marks]; } - else if (settings.filesystem_prefetch_step_bytes && part_stat.approx_size_of_mark) + else if (settings[Setting::filesystem_prefetch_step_bytes] && part_stat.approx_size_of_mark) { part_stat.prefetch_step_marks = std::max( - 1, static_cast(std::round(static_cast(settings.filesystem_prefetch_step_bytes) / part_stat.approx_size_of_mark))); + 1, + static_cast( + std::round(static_cast(settings[Setting::filesystem_prefetch_step_bytes]) / part_stat.approx_size_of_mark))); } part_stat.prefetch_step_marks = std::max(part_stat.prefetch_step_marks, per_part_infos[i]->min_marks_per_task); @@ -406,7 +417,7 @@ void MergeTreePrefetchedReadPool::fillPerThreadTasks(size_t threads, size_t sum_ getPartNameForLogging(parts_ranges[i].data_part), part_stat.sum_marks, part_stat.approx_size_of_mark, - settings.filesystem_prefetch_step_bytes, + settings[Setting::filesystem_prefetch_step_bytes], part_stat.prefetch_step_marks, toString(parts_ranges[i].ranges)); } @@ -419,16 +430,15 @@ void MergeTreePrefetchedReadPool::fillPerThreadTasks(size_t threads, size_t sum_ sum_marks, threads, min_marks_per_thread, - settings.filesystem_prefetches_limit, + settings[Setting::filesystem_prefetches_limit], total_size_approx); - size_t allowed_memory_usage = settings.filesystem_prefetch_max_memory_usage; + size_t allowed_memory_usage = settings[Setting::filesystem_prefetch_max_memory_usage]; if (!allowed_memory_usage) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `filesystem_prefetch_max_memory_usage` must be non-zero"); - std::optional allowed_prefetches_num = settings.filesystem_prefetches_limit - ? std::optional(settings.filesystem_prefetches_limit) - : std::nullopt; + std::optional allowed_prefetches_num + = settings[Setting::filesystem_prefetches_limit] ? std::optional(settings[Setting::filesystem_prefetches_limit]) : std::nullopt; per_thread_tasks.clear(); size_t total_tasks = 0; diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 23fc44cc98a..1e4922757f4 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -19,6 +19,14 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsUInt64 read_backoff_max_throughput; + extern const SettingsUInt64 read_backoff_min_concurrency; + extern const SettingsMilliseconds read_backoff_min_interval_between_events_ms; + extern const SettingsUInt64 read_backoff_min_events; + extern const SettingsMilliseconds read_backoff_min_latency_ms; +} namespace ErrorCodes { @@ -305,12 +313,12 @@ void MergeTreeReadPool::fillPerThreadInfo(size_t threads, size_t sum_marks) } } -MergeTreeReadPool::BackoffSettings::BackoffSettings(const DB::Settings& settings) - : min_read_latency_ms(settings.read_backoff_min_latency_ms.totalMilliseconds()), - max_throughput(settings.read_backoff_max_throughput), - min_interval_between_events_ms(settings.read_backoff_min_interval_between_events_ms.totalMilliseconds()), - min_events(settings.read_backoff_min_events), - min_concurrency(settings.read_backoff_min_concurrency) +MergeTreeReadPool::BackoffSettings::BackoffSettings(const DB::Settings & settings) + : min_read_latency_ms(settings[Setting::read_backoff_min_latency_ms].totalMilliseconds()) + , max_throughput(settings[Setting::read_backoff_max_throughput]) + , min_interval_between_events_ms(settings[Setting::read_backoff_min_interval_between_events_ms].totalMilliseconds()) + , min_events(settings[Setting::read_backoff_min_events]) + , min_concurrency(settings[Setting::read_backoff_min_concurrency]) {} } diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp index 6e9eee3b605..6ce1726398a 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp @@ -6,6 +6,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool merge_tree_determine_task_size_by_prewhere_columns; + extern const SettingsUInt64 merge_tree_min_bytes_per_task_for_remote_reading; +} namespace ErrorCodes { @@ -64,13 +69,13 @@ static size_t calculateMinMarksPerTask( /// We assume that most of the time prewhere does it's job good meaning that lion's share of the rows is filtered out. /// Which means in turn that for most of the rows we will read only the columns from prewhere clause. /// So it makes sense to use only them for the estimation. - const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info + const auto & columns = settings[Setting::merge_tree_determine_task_size_by_prewhere_columns] && prewhere_info ? prewhere_info->prewhere_actions.getRequiredColumnsNames() : columns_to_read; const size_t part_compressed_bytes = getApproxSizeOfPart(*part.data_part, columns); const auto avg_mark_bytes = std::max(part_compressed_bytes / part_marks_count, 1); - const auto min_bytes_per_task = settings.merge_tree_min_bytes_per_task_for_remote_reading; + const auto min_bytes_per_task = settings[Setting::merge_tree_min_bytes_per_task_for_remote_reading]; /// We're taking min here because number of tasks shouldn't be too low - it will make task stealing impossible. /// We also create at least two tasks per thread to have something to steal from a slow thread. const auto heuristic_min_marks diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp index 71d89f9950a..5e2ccc6b9c8 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp @@ -90,6 +90,10 @@ extern const Event ParallelReplicasReadMarks; namespace DB { +namespace Setting +{ + extern const SettingsUInt64 parallel_replicas_mark_segment_size; +} namespace ErrorCodes { @@ -125,7 +129,7 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas( , min_marks_per_task(getMinMarksPerTask(pool_settings.min_marks_for_concurrent_read, per_part_infos)) , mark_segment_size(chooseSegmentSize( log, - context_->getSettingsRef().parallel_replicas_mark_segment_size, + context_->getSettingsRef()[Setting::parallel_replicas_mark_segment_size], min_marks_per_task, pool_settings.threads, pool_settings.sum_marks, diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index dabb6991b0b..6beb0927cbf 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -257,4 +257,8 @@ std::vector MergeTreeSettings::getAllRegisteredNames() const return all_settings; } +std::string_view MergeTreeSettings::resolveName(std::string_view name) +{ + return MergeTreeSettings::Traits::resolveName(name); +} } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index dcb18155114..69954827299 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -280,6 +280,8 @@ struct MergeTreeSettings : public BaseSettings, public void sanityCheck(size_t background_pool_tasks) const; std::vector getAllRegisteredNames() const override; + + static std::string_view resolveName(std::string_view name); }; using MergeTreeSettingsPtr = std::shared_ptr; diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 459c5d5f243..3b87f08c926 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -19,6 +19,11 @@ namespace ErrorCodes namespace DB { +namespace Setting +{ + extern const SettingsBool insert_deduplicate; + extern const SettingsUInt64 max_insert_delayed_streams_for_parallel_write; +} struct MergeTreeSink::DelayedChunk { @@ -140,8 +145,8 @@ void MergeTreeSink::consume(Chunk & chunk) size_t max_insert_delayed_streams_for_parallel_write; - if (settings.max_insert_delayed_streams_for_parallel_write.changed) - max_insert_delayed_streams_for_parallel_write = settings.max_insert_delayed_streams_for_parallel_write; + if (settings[Setting::max_insert_delayed_streams_for_parallel_write].changed) + max_insert_delayed_streams_for_parallel_write = settings[Setting::max_insert_delayed_streams_for_parallel_write]; else if (support_parallel_write) max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE; else @@ -209,7 +214,7 @@ void MergeTreeSink::finishDelayedChunk() auto * deduplication_log = storage.getDeduplicationLog(); - if (settings.insert_deduplicate && deduplication_log) + if (settings[Setting::insert_deduplicate] && deduplication_log) { const String block_id = part->getZeroLevelPartBlockID(partition.block_dedup_token); auto res = deduplication_log->addPart(block_id, part->info); diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 76a02bbd2c4..a2bf45e21aa 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -18,6 +18,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_statistics_optimize; + extern const SettingsUInt64 log_queries_cut_to_length; + extern const SettingsBool move_all_conditions_to_prewhere; + extern const SettingsBool move_primary_key_columns_to_end_of_prewhere; +} /// Conditions like "x = N" are considered good if abs(N) > threshold. /// This is used to assume that condition is likely to have good selectivity. @@ -90,10 +97,11 @@ void MergeTreeWhereOptimizer::optimize(SelectQueryInfo & select_query_info, cons WhereOptimizerContext where_optimizer_context; where_optimizer_context.context = context; where_optimizer_context.array_joined_names = determineArrayJoinedNames(select); - where_optimizer_context.move_all_conditions_to_prewhere = context->getSettingsRef().move_all_conditions_to_prewhere; - where_optimizer_context.move_primary_key_columns_to_end_of_prewhere = context->getSettingsRef().move_primary_key_columns_to_end_of_prewhere; + where_optimizer_context.move_all_conditions_to_prewhere = context->getSettingsRef()[Setting::move_all_conditions_to_prewhere]; + where_optimizer_context.move_primary_key_columns_to_end_of_prewhere + = context->getSettingsRef()[Setting::move_primary_key_columns_to_end_of_prewhere]; where_optimizer_context.is_final = select.final(); - where_optimizer_context.use_statistics = context->getSettingsRef().allow_statistics_optimize; + where_optimizer_context.use_statistics = context->getSettingsRef()[Setting::allow_statistics_optimize]; RPNBuilderTreeContext tree_context(context, std::move(block_with_constants), {} /*prepared_sets*/); RPNBuilderTreeNode node(select.where().get(), tree_context); @@ -109,8 +117,10 @@ void MergeTreeWhereOptimizer::optimize(SelectQueryInfo & select_query_info, cons select.setExpression(ASTSelectQuery::Expression::WHERE, std::move(where_filter_ast)); select.setExpression(ASTSelectQuery::Expression::PREWHERE, std::move(prewhere_filter_ast)); - UInt64 log_queries_cut_to_length = context->getSettingsRef().log_queries_cut_to_length; - LOG_DEBUG(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()->formatForLogging(log_queries_cut_to_length)); + LOG_DEBUG( + log, + "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", + select.prewhere()->formatForLogging(context->getSettingsRef()[Setting::log_queries_cut_to_length])); } MergeTreeWhereOptimizer::FilterActionsOptimizeResult MergeTreeWhereOptimizer::optimize(const ActionsDAG & filter_dag, @@ -121,10 +131,11 @@ MergeTreeWhereOptimizer::FilterActionsOptimizeResult MergeTreeWhereOptimizer::op WhereOptimizerContext where_optimizer_context; where_optimizer_context.context = context; where_optimizer_context.array_joined_names = {}; - where_optimizer_context.move_all_conditions_to_prewhere = context->getSettingsRef().move_all_conditions_to_prewhere; - where_optimizer_context.move_primary_key_columns_to_end_of_prewhere = context->getSettingsRef().move_primary_key_columns_to_end_of_prewhere; + where_optimizer_context.move_all_conditions_to_prewhere = context->getSettingsRef()[Setting::move_all_conditions_to_prewhere]; + where_optimizer_context.move_primary_key_columns_to_end_of_prewhere + = context->getSettingsRef()[Setting::move_primary_key_columns_to_end_of_prewhere]; where_optimizer_context.is_final = is_final; - where_optimizer_context.use_statistics = context->getSettingsRef().allow_statistics_optimize; + where_optimizer_context.use_statistics = context->getSettingsRef()[Setting::allow_statistics_optimize]; RPNBuilderTreeContext tree_context(context); RPNBuilderTreeNode node(&filter_dag.findInOutputs(filter_column_name), tree_context); diff --git a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp index 10461eb5942..13386d94773 100644 --- a/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp +++ b/src/Storages/MergeTree/MutatePlainMergeTreeTask.cpp @@ -7,6 +7,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool enable_sharing_sets_for_mutations; +} namespace ErrorCodes { @@ -52,7 +56,7 @@ void MutatePlainMergeTreeTask::prepare() std::move(profile_counters_snapshot)); }; - if (task_context->getSettingsRef().enable_sharing_sets_for_mutations) + if (task_context->getSettingsRef()[Setting::enable_sharing_sets_for_mutations]) { /// If we have a prepared sets cache for this mutations, we will use it. auto mutation_id = future_part->part_info.mutation; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 03c7c1b3024..a92c743304f 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -55,6 +55,11 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_rows; +} namespace ErrorCodes { @@ -1169,7 +1174,7 @@ void PartMergerWriter::prepare() for (size_t i = 0, size = ctx->projections_to_build.size(); i < size; ++i) { // We split the materialization into multiple stages similar to the process of INSERT SELECT query. - projection_squashes.emplace_back(ctx->updated_header, settings.min_insert_block_size_rows, settings.min_insert_block_size_bytes); + projection_squashes.emplace_back(ctx->updated_header, settings[Setting::min_insert_block_size_rows], settings[Setting::min_insert_block_size_bytes]); } existing_rows_count = 0; diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 6e963066f39..056440fcda1 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -24,6 +24,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} namespace ErrorCodes { @@ -131,7 +135,7 @@ std::string RPNBuilderTreeNode::getColumnName() const if (ast_node) return ast_node->getColumnNameWithoutAlias(); else - return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer); + return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings()[Setting::allow_experimental_analyzer]); } std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const @@ -144,7 +148,7 @@ std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const } else { - return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer, true /*legacy*/); + return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings()[Setting::allow_experimental_analyzer], true /*legacy*/); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index 2f0b71b977e..69c84d2c86a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -28,6 +28,16 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsFloat insert_keeper_fault_injection_probability; + extern const SettingsUInt64 insert_keeper_fault_injection_seed; + extern const SettingsUInt64 insert_keeper_max_retries; + extern const SettingsUInt64 insert_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 insert_keeper_retry_max_backoff_ms; + extern const SettingsUInt64 max_insert_delayed_streams_for_parallel_write; + extern const SettingsBool optimize_on_insert; +} namespace FailPoints { @@ -180,7 +190,9 @@ size_t ReplicatedMergeTreeSinkImpl::checkQuorumPrecondition(const ZooKeeperRetriesControl quorum_retries_ctl( "checkQuorumPrecondition", log, - {settings.insert_keeper_max_retries, settings.insert_keeper_retry_initial_backoff_ms, settings.insert_keeper_retry_max_backoff_ms}, + {settings[Setting::insert_keeper_max_retries], + settings[Setting::insert_keeper_retry_initial_backoff_ms], + settings[Setting::insert_keeper_retry_max_backoff_ms]}, context->getProcessListElement()); quorum_retries_ctl.retryLoop( [&]() @@ -278,8 +290,8 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk & chunk) const auto & settings = context->getSettingsRef(); ZooKeeperWithFaultInjectionPtr zookeeper = ZooKeeperWithFaultInjection::createInstance( - settings.insert_keeper_fault_injection_probability, - settings.insert_keeper_fault_injection_seed, + settings[Setting::insert_keeper_fault_injection_probability], + settings[Setting::insert_keeper_fault_injection_seed], storage.getZooKeeper(), "ReplicatedMergeTreeSink::consume", log); @@ -341,7 +353,7 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk & chunk) if constexpr (async_insert) { /// we copy everything but offsets which we move because they are only used by async insert - if (settings.optimize_on_insert && storage.writer.getMergingMode() != MergeTreeData::MergingParams::Mode::Ordinary) + if (settings[Setting::optimize_on_insert] && storage.writer.getMergingMode() != MergeTreeData::MergingParams::Mode::Ordinary) unmerged_block.emplace(Block(current_block.block), Row(current_block.partition), std::move(current_block.offsets), std::move(current_block.tokens)); } @@ -393,8 +405,8 @@ void ReplicatedMergeTreeSinkImpl::consume(Chunk & chunk) UInt64 elapsed_ns = watch.elapsed(); size_t max_insert_delayed_streams_for_parallel_write; - if (settings.max_insert_delayed_streams_for_parallel_write.changed) - max_insert_delayed_streams_for_parallel_write = settings.max_insert_delayed_streams_for_parallel_write; + if (settings[Setting::max_insert_delayed_streams_for_parallel_write].changed) + max_insert_delayed_streams_for_parallel_write = settings[Setting::max_insert_delayed_streams_for_parallel_write]; else if (support_parallel_write) max_insert_delayed_streams_for_parallel_write = DEFAULT_DELAYED_STREAMS_FOR_PARALLEL_WRITE; else @@ -681,7 +693,9 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: ZooKeeperRetriesControl retries_ctl( "commitPart", log, - {settings.insert_keeper_max_retries, settings.insert_keeper_retry_initial_backoff_ms, settings.insert_keeper_retry_max_backoff_ms}, + {settings[Setting::insert_keeper_max_retries], + settings[Setting::insert_keeper_retry_initial_backoff_ms], + settings[Setting::insert_keeper_retry_max_backoff_ms]}, context->getProcessListElement()); auto resolve_duplicate_stage = [&] () -> CommitRetryContext::Stages @@ -1170,8 +1184,8 @@ void ReplicatedMergeTreeSinkImpl::onFinish() const auto & settings = context->getSettingsRef(); ZooKeeperWithFaultInjectionPtr zookeeper = ZooKeeperWithFaultInjection::createInstance( - settings.insert_keeper_fault_injection_probability, - settings.insert_keeper_fault_injection_seed, + settings[Setting::insert_keeper_fault_injection_probability], + settings[Setting::insert_keeper_fault_injection_seed], storage.getZooKeeper(), "ReplicatedMergeTreeSink::onFinish", log); diff --git a/src/Storages/MergeTree/VectorSimilarityCondition.cpp b/src/Storages/MergeTree/VectorSimilarityCondition.cpp index 641b0037e7b..a8c61ae4894 100644 --- a/src/Storages/MergeTree/VectorSimilarityCondition.cpp +++ b/src/Storages/MergeTree/VectorSimilarityCondition.cpp @@ -13,6 +13,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_limit_for_ann_queries; +} namespace ErrorCodes { @@ -55,7 +59,7 @@ VectorSimilarityCondition::Info::DistanceFunction stringToDistanceFunction(const VectorSimilarityCondition::VectorSimilarityCondition(const SelectQueryInfo & query_info, ContextPtr context) : block_with_constants(KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)) , index_granularity(context->getMergeTreeSettings().index_granularity) - , max_limit_for_ann_queries(context->getSettingsRef().max_limit_for_ann_queries) + , max_limit_for_ann_queries(context->getSettingsRef()[Setting::max_limit_for_ann_queries]) , index_is_useful(checkQueryStructure(query_info)) {} diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index c7ff266f30f..380a1300990 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -28,6 +28,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_deprecated_syntax_for_merge_tree; + extern const SettingsBool allow_suspicious_primary_key; + extern const SettingsBool allow_suspicious_ttl_expressions; + extern const SettingsBool create_table_empty_primary_key_by_default; + extern const SettingsUInt64 database_replicated_allow_replicated_engine_arguments; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -277,7 +286,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( if (has_valid_arguments) { - if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 0) + if (!query.attach && is_replicated_database && local_context->getSettingsRef()[Setting::database_replicated_allow_replicated_engine_arguments] == 0) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "It's not allowed to specify explicit zookeeper_path and replica_name " @@ -285,7 +294,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( "specify them explicitly, enable setting " "database_replicated_allow_replicated_engine_arguments."); } - else if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 1) + else if (!query.attach && is_replicated_database && local_context->getSettingsRef()[Setting::database_replicated_allow_replicated_engine_arguments] == 1) { LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "It's not recommended to explicitly specify " "zookeeper_path and replica_name in ReplicatedMergeTree arguments"); @@ -305,7 +314,7 @@ static void extractZooKeeperPathAndReplicaNameFromEngineArgs( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica name must be a string literal{}", verbose_help_message); - if (!query.attach && is_replicated_database && local_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments == 2) + if (!query.attach && is_replicated_database && local_context->getSettingsRef()[Setting::database_replicated_allow_replicated_engine_arguments] == 2) { LOG_WARNING(&Poco::Logger::get("registerStorageMergeTree"), "Replacing user-provided ZooKeeper path and replica name ({}, {}) " "with default arguments", zookeeper_path, replica_name); @@ -543,7 +552,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// In new syntax argument can be literal or identifier or array/tuple of identifiers. evaluateEngineArgs(engine_args, args.getLocalContext()); } - else if (args.mode <= LoadingStrictnessLevel::CREATE && !local_settings.allow_deprecated_syntax_for_merge_tree) + else if (args.mode <= LoadingStrictnessLevel::CREATE && !local_settings[Setting::allow_deprecated_syntax_for_merge_tree]) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "This syntax for *MergeTree engine is deprecated. " "Use extended storage definition syntax with ORDER BY/PRIMARY KEY clause. " @@ -676,7 +685,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (!args.storage_def->order_by) { - if (local_settings.create_table_empty_primary_key_by_default) + if (local_settings[Setting::create_table_empty_primary_key_by_default]) { args.storage_def->set(args.storage_def->order_by, makeASTFunction("tuple")); } @@ -697,7 +706,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// column if sorting key will be changed. metadata.sorting_key = KeyDescription::getSortingKeyFromAST( args.storage_def->order_by->ptr(), metadata.columns, context, merging_param_key_arg); - if (!local_settings.allow_suspicious_primary_key && args.mode <= LoadingStrictnessLevel::CREATE) + if (!local_settings[Setting::allow_suspicious_primary_key] && args.mode <= LoadingStrictnessLevel::CREATE) MergeTreeData::verifySortingKey(metadata.sorting_key); /// If primary key explicitly defined, than get it from AST @@ -723,7 +732,8 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (args.storage_def->sample_by) metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, context); - bool allow_suspicious_ttl = LoadingStrictnessLevel::SECONDARY_CREATE <= args.mode || local_settings.allow_suspicious_ttl_expressions; + bool allow_suspicious_ttl + = LoadingStrictnessLevel::SECONDARY_CREATE <= args.mode || local_settings[Setting::allow_suspicious_ttl_expressions]; if (args.storage_def->ttl_table) { @@ -811,7 +821,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// column if sorting key will be changed. metadata.sorting_key = KeyDescription::getSortingKeyFromAST(engine_args[arg_num], metadata.columns, context, merging_param_key_arg); - if (!local_settings.allow_suspicious_primary_key && args.mode <= LoadingStrictnessLevel::CREATE) + if (!local_settings[Setting::allow_suspicious_primary_key] && args.mode <= LoadingStrictnessLevel::CREATE) MergeTreeData::verifySortingKey(metadata.sorting_key); /// In old syntax primary_key always equals to sorting key. diff --git a/src/Storages/MySQL/MySQLSettings.cpp b/src/Storages/MySQL/MySQLSettings.cpp index 1c88e4a94e9..ee0378a2403 100644 --- a/src/Storages/MySQL/MySQLSettings.cpp +++ b/src/Storages/MySQL/MySQLSettings.cpp @@ -11,6 +11,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsMySQLDataTypesSupport mysql_datatypes_support_level; +} namespace ErrorCodes { @@ -54,10 +58,10 @@ void MySQLSettings::loadFromQueryContext(ContextPtr context, ASTStorage & storag const Settings & settings = context->getQueryContext()->getSettingsRef(); - if (settings.mysql_datatypes_support_level.value != mysql_datatypes_support_level.value) + if (settings[Setting::mysql_datatypes_support_level].value != mysql_datatypes_support_level.value) { static constexpr auto setting_name = "mysql_datatypes_support_level"; - set(setting_name, settings.mysql_datatypes_support_level.toString()); + set(setting_name, settings[Setting::mysql_datatypes_support_level].toString()); if (!storage_def.settings) { @@ -71,7 +75,7 @@ void MySQLSettings::loadFromQueryContext(ContextPtr context, ASTStorage & storag changes.begin(), changes.end(), [](const SettingChange & c) { return c.name == setting_name; })) { - changes.push_back(SettingChange{setting_name, settings.mysql_datatypes_support_level.toString()}); + changes.push_back(SettingChange{setting_name, settings[Setting::mysql_datatypes_support_level].toString()}); } } } diff --git a/src/Storages/NATS/NATSSettings.cpp b/src/Storages/NATS/NATSSettings.cpp index ffdb79247d2..c3174ccb9bb 100644 --- a/src/Storages/NATS/NATSSettings.cpp +++ b/src/Storages/NATS/NATSSettings.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/NATS/NATSSettings.h b/src/Storages/NATS/NATSSettings.h index 3273a5ff065..bb756d38559 100644 --- a/src/Storages/NATS/NATSSettings.h +++ b/src/Storages/NATS/NATSSettings.h @@ -1,8 +1,9 @@ #pragma once #include -#include +#include #include +#include namespace DB { diff --git a/src/Storages/NATS/NATSSource.cpp b/src/Storages/NATS/NATSSource.cpp index 54f479faacc..40068be92d4 100644 --- a/src/Storages/NATS/NATSSource.cpp +++ b/src/Storages/NATS/NATSSource.cpp @@ -1,13 +1,18 @@ #include +#include #include +#include #include #include #include -#include namespace DB { +namespace Setting +{ + extern const SettingsMilliseconds rabbitmq_max_wait_ms; +} static std::pair getHeaders(const StorageSnapshotPtr & storage_snapshot) { @@ -86,7 +91,7 @@ Chunk NATSSource::generate() { if (!consumer) { - auto timeout = std::chrono::milliseconds(context->getSettingsRef().rabbitmq_max_wait_ms.totalMilliseconds()); + auto timeout = std::chrono::milliseconds(context->getSettingsRef()[Setting::rabbitmq_max_wait_ms].totalMilliseconds()); consumer = storage.popConsumer(timeout); consumer->subscribe(); } diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index 9d728c3395f..8051189ca33 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include #include @@ -25,11 +25,20 @@ #include #include #include +#include #include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsMilliseconds stream_flush_interval_ms; + extern const SettingsBool stream_like_engine_allow_direct_select; + extern const SettingsString stream_like_engine_insert_queue; + extern const SettingsUInt64 output_format_avro_rows_in_file; +} static const uint32_t QUEUE_SIZE = 100000; static const auto RESCHEDULE_MS = 500; @@ -299,7 +308,7 @@ void StorageNATS::deactivateTask(BackgroundSchedulePool::TaskHolder & task, bool size_t StorageNATS::getMaxBlockSize() const { return nats_settings->nats_max_block_size.changed ? nats_settings->nats_max_block_size.value - : (getContext()->getSettingsRef().max_insert_block_size.value / num_consumers); + : (getContext()->getSettingsRef()[Setting::max_insert_block_size].value / num_consumers); } @@ -319,7 +328,7 @@ void StorageNATS::read( if (num_created_consumers == 0) return; - if (!local_context->getSettingsRef().stream_like_engine_allow_direct_select) + if (!local_context->getSettingsRef()[Setting::stream_like_engine_allow_direct_select]) throw Exception( ErrorCodes::QUERY_NOT_ALLOWED, "Direct select is not allowed. To enable use setting `stream_like_engine_allow_direct_select`"); @@ -379,9 +388,9 @@ void StorageNATS::read( SinkToStoragePtr StorageNATS::write(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, bool /*async_insert*/) { auto modified_context = addSettings(local_context); - std::string subject = modified_context->getSettingsRef().stream_like_engine_insert_queue.changed - ? modified_context->getSettingsRef().stream_like_engine_insert_queue.value - : ""; + std::string subject = modified_context->getSettingsRef()[Setting::stream_like_engine_insert_queue].changed + ? modified_context->getSettingsRef()[Setting::stream_like_engine_insert_queue].value + : ""; if (subject.empty()) { if (subjects.size() > 1) @@ -407,8 +416,8 @@ SinkToStoragePtr StorageNATS::write(const ASTPtr &, const StorageMetadataPtr & m auto producer = std::make_unique(configuration, subject, shutdown_called, log); size_t max_rows = max_rows_per_message; /// Need for backward compatibility. - if (format_name == "Avro" && local_context->getSettingsRef().output_format_avro_rows_in_file.changed) - max_rows = local_context->getSettingsRef().output_format_avro_rows_in_file.value; + if (format_name == "Avro" && local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].changed) + max_rows = local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].value; return std::make_shared( metadata_snapshot->getSampleBlockNonMaterialized(), getFormatName(), max_rows, std::move(producer), getName(), modified_context);} @@ -676,7 +685,7 @@ bool StorageNATS::streamToViews() Poco::Timespan max_execution_time = nats_settings->nats_flush_interval_ms.changed ? nats_settings->nats_flush_interval_ms - : getContext()->getSettingsRef().stream_flush_interval_ms; + : getContext()->getSettingsRef()[Setting::stream_flush_interval_ms]; source->setTimeLimit(max_execution_time); } diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index b58aed573ae..eb4cb317f49 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -1,16 +1,23 @@ #include "NamedCollectionsHelpers.h" #include -#include -#include #include #include -#include -#include #include +#include #include +#include +#include +#include +#include + +#include namespace DB { +namespace Setting +{ + extern const SettingsBool allow_named_collection_override_by_default; +} namespace ErrorCodes { @@ -118,7 +125,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( if (asts.size() == 1) return collection_copy; - const auto allow_override_by_default = context->getSettingsRef().allow_named_collection_override_by_default; + const auto allow_override_by_default = context->getSettingsRef()[Setting::allow_named_collection_override_by_default]; for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { @@ -163,7 +170,7 @@ MutableNamedCollectionPtr tryGetNamedCollectionWithOverrides( Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_prefix, keys); - const auto allow_override_by_default = context->getSettingsRef().allow_named_collection_override_by_default; + const auto allow_override_by_default = context->getSettingsRef()[Setting::allow_named_collection_override_by_default]; for (const auto & key : keys) { if (collection_copy->isOverridable(key, allow_override_by_default)) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 8121f389a8d..d98c4753b1a 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -20,6 +20,18 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool azure_create_new_file_on_insert; + extern const SettingsBool azure_ignore_file_doesnt_exist; + extern const SettingsUInt64 azure_list_object_keys_size; + extern const SettingsBool azure_skip_empty_files; + extern const SettingsBool azure_throw_on_zero_files_match; + extern const SettingsBool azure_truncate_on_insert; + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsBool schema_inference_use_cache_for_azure; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -62,14 +74,14 @@ StorageObjectStorage::QuerySettings StorageAzureConfiguration::getQuerySettings( { const auto & settings = context->getSettingsRef(); return StorageObjectStorage::QuerySettings{ - .truncate_on_insert = settings.azure_truncate_on_insert, - .create_new_file_on_insert = settings.azure_create_new_file_on_insert, - .schema_inference_use_cache = settings.schema_inference_use_cache_for_azure, - .schema_inference_mode = settings.schema_inference_mode, - .skip_empty_files = settings.azure_skip_empty_files, - .list_object_keys_size = settings.azure_list_object_keys_size, - .throw_on_zero_files_match = settings.azure_throw_on_zero_files_match, - .ignore_non_existent_file = settings.azure_ignore_file_doesnt_exist, + .truncate_on_insert = settings[Setting::azure_truncate_on_insert], + .create_new_file_on_insert = settings[Setting::azure_create_new_file_on_insert], + .schema_inference_use_cache = settings[Setting::schema_inference_use_cache_for_azure], + .schema_inference_mode = settings[Setting::schema_inference_mode], + .skip_empty_files = settings[Setting::azure_skip_empty_files], + .list_object_keys_size = settings[Setting::azure_list_object_keys_size], + .throw_on_zero_files_match = settings[Setting::azure_throw_on_zero_files_match], + .ignore_non_existent_file = settings[Setting::azure_ignore_file_doesnt_exist], }; } diff --git a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h index 4b21bdb0494..a17fd163253 100644 --- a/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h +++ b/src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h @@ -4,7 +4,6 @@ #if USE_AVRO -#include #include #include #include diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 9b9d92e282c..ffc4dd09a3a 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -35,6 +35,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool iceberg_engine_ignore_schema_evolution; +} namespace ErrorCodes { @@ -394,7 +398,8 @@ DataLakeMetadataPtr IcebergMetadata::create( Poco::JSON::Object::Ptr object = json.extract(); auto format_version = object->getValue("format-version"); - auto [schema, schema_id] = parseTableSchema(object, format_version, local_context->getSettingsRef().iceberg_engine_ignore_schema_evolution); + auto [schema, schema_id] + = parseTableSchema(object, format_version, local_context->getSettingsRef()[Setting::iceberg_engine_ignore_schema_evolution]); auto current_snapshot_id = object->getValue("current-snapshot-id"); auto snapshots = object->get("snapshots").extract(); @@ -493,7 +498,8 @@ Strings IcebergMetadata::getDataFiles() const Poco::JSON::Parser parser; Poco::Dynamic::Var json = parser.parse(schema_json_string); Poco::JSON::Object::Ptr schema_object = json.extract(); - if (!context->getSettingsRef().iceberg_engine_ignore_schema_evolution && schema_object->getValue("schema-id") != current_schema_id) + if (!context->getSettingsRef()[Setting::iceberg_engine_ignore_schema_evolution] + && schema_object->getValue("schema-id") != current_schema_id) throw Exception( ErrorCodes::UNSUPPORTED_METHOD, "Cannot read Iceberg table: the table schema has been changed at least 1 time, reading tables with evolved schema is not " diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 9b5bbdeacc1..6bee4154b2f 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -20,6 +20,19 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool hdfs_create_new_file_on_insert; + extern const SettingsBool hdfs_ignore_file_doesnt_exist; + extern const SettingsUInt64 hdfs_replication; + extern const SettingsBool hdfs_skip_empty_files; + extern const SettingsBool hdfs_throw_on_zero_files_match; + extern const SettingsBool hdfs_truncate_on_insert; + extern const SettingsUInt64 remote_read_min_bytes_for_seek; + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsBool schema_inference_use_cache_for_hdfs; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -48,10 +61,7 @@ ObjectStoragePtr StorageHDFSConfiguration::createObjectStorage( /// NOLINT { assertInitialized(); const auto & settings = context->getSettingsRef(); - auto hdfs_settings = std::make_unique( - settings.remote_read_min_bytes_for_seek, - settings.hdfs_replication - ); + auto hdfs_settings = std::make_unique(settings[Setting::remote_read_min_bytes_for_seek], settings[Setting::hdfs_replication]); return std::make_shared( url, std::move(hdfs_settings), context->getConfigRef(), /* lazy_initialize */true); } @@ -71,14 +81,14 @@ StorageObjectStorage::QuerySettings StorageHDFSConfiguration::getQuerySettings(c { const auto & settings = context->getSettingsRef(); return StorageObjectStorage::QuerySettings{ - .truncate_on_insert = settings.hdfs_truncate_on_insert, - .create_new_file_on_insert = settings.hdfs_create_new_file_on_insert, - .schema_inference_use_cache = settings.schema_inference_use_cache_for_hdfs, - .schema_inference_mode = settings.schema_inference_mode, - .skip_empty_files = settings.hdfs_skip_empty_files, + .truncate_on_insert = settings[Setting::hdfs_truncate_on_insert], + .create_new_file_on_insert = settings[Setting::hdfs_create_new_file_on_insert], + .schema_inference_use_cache = settings[Setting::schema_inference_use_cache_for_hdfs], + .schema_inference_mode = settings[Setting::schema_inference_mode], + .skip_empty_files = settings[Setting::hdfs_skip_empty_files], .list_object_keys_size = 0, /// HDFS does not support listing in batches. - .throw_on_zero_files_match = settings.hdfs_throw_on_zero_files_match, - .ignore_non_existent_file = settings.hdfs_ignore_file_doesnt_exist, + .throw_on_zero_files_match = settings[Setting::hdfs_throw_on_zero_files_match], + .ignore_non_existent_file = settings[Setting::hdfs_ignore_file_doesnt_exist], }; } diff --git a/src/Storages/ObjectStorage/Local/Configuration.cpp b/src/Storages/ObjectStorage/Local/Configuration.cpp index 0554b9c317c..bae84714273 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.cpp +++ b/src/Storages/ObjectStorage/Local/Configuration.cpp @@ -8,6 +8,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool engine_file_skip_empty_files; + extern const SettingsBool engine_file_truncate_on_insert; + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsBool schema_inference_use_cache_for_file; +} namespace ErrorCodes { @@ -64,11 +71,11 @@ StorageObjectStorage::QuerySettings StorageLocalConfiguration::getQuerySettings( { const auto & settings = context->getSettingsRef(); return StorageObjectStorage::QuerySettings{ - .truncate_on_insert = settings.engine_file_truncate_on_insert, + .truncate_on_insert = settings[Setting::engine_file_truncate_on_insert], .create_new_file_on_insert = false, - .schema_inference_use_cache = settings.schema_inference_use_cache_for_file, - .schema_inference_mode = settings.schema_inference_mode, - .skip_empty_files = settings.engine_file_skip_empty_files, + .schema_inference_use_cache = settings[Setting::schema_inference_use_cache_for_file], + .schema_inference_mode = settings[Setting::schema_inference_mode], + .skip_empty_files = settings[Setting::engine_file_skip_empty_files], .list_object_keys_size = 0, .throw_on_zero_files_match = false, .ignore_non_existent_file = false}; diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp index fadf683fce7..f0f3de97652 100644 --- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp +++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp @@ -2,11 +2,17 @@ #include #include #include +#include #include namespace DB { +namespace Setting +{ + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsInt64 zstd_window_log_max; +} namespace ErrorCodes { @@ -148,9 +154,9 @@ std::unique_ptr ReadBufferIterator::recreateLastReadBuffer() auto impl = object_storage->readObject(StoredObject(path), context->getReadSettings()); const auto compression_method = chooseCompressionMethod(current_object_info->getFileName(), configuration->compression_method); - const auto zstd_window_log_max = static_cast(context->getSettingsRef().zstd_window_log_max); + const auto zstd_window = static_cast(context->getSettingsRef()[Setting::zstd_window_log_max]); - return wrapReadBufferWithCompressionMethod(std::move(impl), compression_method, zstd_window_log_max); + return wrapReadBufferWithCompressionMethod(std::move(impl), compression_method, zstd_window); } ReadBufferIterator::Data ReadBufferIterator::next() @@ -174,7 +180,7 @@ ReadBufferIterator::Data ReadBufferIterator::next() } /// For default mode check cached columns for currently read keys on first iteration. - if (first && getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::DEFAULT) + if (first && getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::DEFAULT) { if (auto cached_columns = tryGetColumnsFromCache(read_keys.begin(), read_keys.end())) { @@ -234,7 +240,7 @@ ReadBufferIterator::Data ReadBufferIterator::next() } /// Check new files in schema cache if schema inference mode is default. - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::DEFAULT) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::DEFAULT) { auto columns_from_cache = tryGetColumnsFromCache(read_keys.begin() + prev_read_keys_size, read_keys.end()); if (columns_from_cache) @@ -249,7 +255,7 @@ ReadBufferIterator::Data ReadBufferIterator::next() continue; /// In union mode, check cached columns only for current key. - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::UNION) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::UNION) { ObjectInfos objects{current_object_info}; if (auto columns_from_cache = tryGetColumnsFromCache(objects.begin(), objects.end())) @@ -283,9 +289,7 @@ ReadBufferIterator::Data ReadBufferIterator::next() first = false; read_buf = wrapReadBufferWithCompressionMethod( - std::move(read_buf), - compression_method, - static_cast(getContext()->getSettingsRef().zstd_window_log_max)); + std::move(read_buf), compression_method, static_cast(getContext()->getSettingsRef()[Setting::zstd_window_log_max])); return {std::move(read_buf), std::nullopt, format}; } diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 56bc6ea2f61..24e503fd5b3 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -1,6 +1,7 @@ #include #if USE_AWS_S3 +#include #include #include #include @@ -18,9 +19,24 @@ #include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsBool allow_archive_path_syntax; + extern const SettingsBool s3_create_new_file_on_insert; + extern const SettingsBool s3_ignore_file_doesnt_exist; + extern const SettingsUInt64 s3_list_object_keys_size; + extern const SettingsBool s3_skip_empty_files; + extern const SettingsBool s3_truncate_on_insert; + extern const SettingsBool s3_throw_on_zero_files_match; + extern const SettingsBool s3_validate_request_settings; + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsBool schema_inference_use_cache_for_s3; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -90,14 +106,14 @@ StorageObjectStorage::QuerySettings StorageS3Configuration::getQuerySettings(con { const auto & settings = context->getSettingsRef(); return StorageObjectStorage::QuerySettings{ - .truncate_on_insert = settings.s3_truncate_on_insert, - .create_new_file_on_insert = settings.s3_create_new_file_on_insert, - .schema_inference_use_cache = settings.schema_inference_use_cache_for_s3, - .schema_inference_mode = settings.schema_inference_mode, - .skip_empty_files = settings.s3_skip_empty_files, - .list_object_keys_size = settings.s3_list_object_keys_size, - .throw_on_zero_files_match = settings.s3_throw_on_zero_files_match, - .ignore_non_existent_file = settings.s3_ignore_file_doesnt_exist, + .truncate_on_insert = settings[Setting::s3_truncate_on_insert], + .create_new_file_on_insert = settings[Setting::s3_create_new_file_on_insert], + .schema_inference_use_cache = settings[Setting::schema_inference_use_cache_for_s3], + .schema_inference_mode = settings[Setting::schema_inference_mode], + .skip_empty_files = settings[Setting::s3_skip_empty_files], + .list_object_keys_size = settings[Setting::s3_list_object_keys_size], + .throw_on_zero_files_match = settings[Setting::s3_throw_on_zero_files_match], + .ignore_non_existent_file = settings[Setting::s3_ignore_file_doesnt_exist], }; } @@ -108,8 +124,7 @@ ObjectStoragePtr StorageS3Configuration::createObjectStorage(ContextPtr context, const auto & config = context->getConfigRef(); const auto & settings = context->getSettingsRef(); - auto s3_settings = getSettings( - config, "s3"/* config_prefix */, context, url.uri_str, settings.s3_validate_request_settings); + auto s3_settings = getSettings(config, "s3" /* config_prefix */, context, url.uri_str, settings[Setting::s3_validate_request_settings]); if (auto endpoint_settings = context->getStorageS3Settings().getSettings(url.uri.toString(), context->getUserName())) { @@ -147,9 +162,9 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect auto filename = collection.getOrDefault("filename", ""); if (!filename.empty()) - url = S3::URI(std::filesystem::path(collection.get("url")) / filename, settings.allow_archive_path_syntax); + url = S3::URI(std::filesystem::path(collection.get("url")) / filename, settings[Setting::allow_archive_path_syntax]); else - url = S3::URI(collection.get("url"), settings.allow_archive_path_syntax); + url = S3::URI(collection.get("url"), settings[Setting::allow_archive_path_syntax]); auth_settings.access_key_id = collection.getOrDefault("access_key_id", ""); auth_settings.secret_access_key = collection.getOrDefault("secret_access_key", ""); @@ -338,7 +353,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } /// This argument is always the first - url = S3::URI(checkAndGetLiteralArgument(args[0], "url"), context->getSettingsRef().allow_archive_path_syntax); + url = S3::URI(checkAndGetLiteralArgument(args[0], "url"), context->getSettingsRef()[Setting::allow_archive_path_syntax]); if (engine_args_to_idx.contains("format")) { diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index f0686c144ba..bc27820707c 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -26,6 +26,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsMaxThreads max_threads; + extern const SettingsBool optimize_count_from_files; + extern const SettingsBool use_hive_partitioning; +} namespace ErrorCodes { @@ -41,7 +47,7 @@ String StorageObjectStorage::getPathSample(StorageInMemoryMetadata metadata, Con query_settings.throw_on_zero_files_match = false; bool local_distributed_processing = distributed_processing; - if (context->getSettingsRef().use_hive_partitioning) + if (context->getSettingsRef()[Setting::use_hive_partitioning]) local_distributed_processing = false; auto file_iterator = StorageObjectStorageSource::createFileIterator( @@ -91,7 +97,7 @@ StorageObjectStorage::StorageObjectStorage( metadata.setConstraints(constraints_); metadata.setComment(comment); - if (sample_path.empty() && context->getSettingsRef().use_hive_partitioning) + if (sample_path.empty() && context->getSettingsRef()[Setting::use_hive_partitioning]) sample_path = getPathSample(metadata, context); setVirtuals(VirtualColumnUtils::getVirtualsForFileLikeStorage(metadata.columns, context, sample_path, format_settings)); @@ -177,7 +183,7 @@ public: Pipes pipes; auto context = getContext(); - const size_t max_threads = context->getSettingsRef().max_threads; + const size_t max_threads = context->getSettingsRef()[Setting::max_threads]; size_t estimated_keys_count = iterator_wrapper->estimatedKeysCount(); if (estimated_keys_count > 1) @@ -267,7 +273,7 @@ void StorageObjectStorage::read( const auto read_from_format_info = prepareReadingFromFormat( column_names, storage_snapshot, supportsSubsetOfColumns(local_context), local_context); const bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) - && local_context->getSettingsRef().optimize_count_from_files; + && local_context->getSettingsRef()[Setting::optimize_count_from_files]; auto read_step = std::make_unique( object_storage, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index d712e4eec20..0bb6660191f 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -15,6 +15,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool use_hive_partitioning; +} namespace ErrorCodes { @@ -65,7 +69,7 @@ StorageObjectStorageCluster::StorageObjectStorageCluster( metadata.setColumns(columns); metadata.setConstraints(constraints_); - if (sample_path.empty() && context_->getSettingsRef().use_hive_partitioning) + if (sample_path.empty() && context_->getSettingsRef()[Setting::use_hive_partitioning]) sample_path = getPathSample(metadata, context_); setVirtuals(VirtualColumnUtils::getVirtualsForFileLikeStorage(metadata.columns, context_, sample_path)); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp index 33c96fdfae9..7d9b69dfc09 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp @@ -8,6 +8,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 output_format_compression_level; + extern const SettingsUInt64 output_format_compression_zstd_window_log; +} + namespace ErrorCodes { extern const int CANNOT_PARSE_TEXT; @@ -32,10 +38,10 @@ StorageObjectStorageSink::StorageObjectStorageSink( StoredObject(path), WriteMode::Rewrite, std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, context->getWriteSettings()); write_buf = wrapWriteBufferWithCompressionMethod( - std::move(buffer), - chosen_compression_method, - static_cast(settings.output_format_compression_level), - static_cast(settings.output_format_compression_zstd_window_log)); + std::move(buffer), + chosen_compression_method, + static_cast(settings[Setting::output_format_compression_level]), + static_cast(settings[Setting::output_format_compression_zstd_window_log])); writer = FormatFactory::instance().getOutputFormatParallelIfPossible( configuration->format, *write_buf, sample_block, context, format_settings_); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 6ad453bc0de..641b43e57d6 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -31,6 +31,12 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_download_buffer_size; + extern const SettingsMaxThreads max_threads; + extern const SettingsBool use_cache_for_count_from_files; +} namespace ErrorCodes { @@ -109,9 +115,7 @@ std::shared_ptr StorageObjectStorageSourc std::function file_progress_callback) { if (distributed_processing) - return std::make_shared( - local_context->getReadTaskCallback(), - local_context->getSettingsRef().max_threads); + return std::make_shared(local_context->getReadTaskCallback(), local_context->getSettingsRef()[Setting::max_threads]); if (configuration->isNamespaceWithGlobs()) throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -238,7 +242,7 @@ Chunk StorageObjectStorageSource::generate() return chunk; } - if (reader.getInputFormat() && getContext()->getSettingsRef().use_cache_for_count_from_files) + if (reader.getInputFormat() && getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) addNumRowsToCache(*reader.getObjectInfo(), total_rows_in_file); total_rows_in_file = 0; @@ -332,10 +336,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade return schema_cache->tryGetNumRows(cache_key, get_last_mod_time); }; - std::optional num_rows_from_cache = need_only_count - && context_->getSettingsRef().use_cache_for_count_from_files - ? try_get_num_rows_from_cache() - : std::nullopt; + std::optional num_rows_from_cache + = need_only_count && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] ? try_get_num_rows_from_cache() : std::nullopt; if (num_rows_from_cache) { @@ -426,7 +428,7 @@ std::unique_ptr StorageObjectStorageSource::createReadBuffer( /// FIXME: Changing this setting to default value breaks something around parquet reading read_settings.remote_read_min_bytes_for_seek = read_settings.remote_fs_buffer_size; - const bool object_too_small = object_size <= 2 * context_->getSettingsRef().max_download_buffer_size; + const bool object_too_small = object_size <= 2 * context_->getSettingsRef()[Setting::max_download_buffer_size]; const bool use_prefetch = object_too_small && read_settings.remote_fs_method == RemoteFSReadMethod::threadpool; read_settings.remote_fs_method = use_prefetch ? RemoteFSReadMethod::threadpool : RemoteFSReadMethod::read; /// User's object may change, don't cache it. diff --git a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp index b5f4cf5bb54..d0cacc29adf 100644 --- a/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/registerStorageObjectStorage.cpp @@ -1,8 +1,9 @@ +#include #include #include #include -#include #include +#include #include #include @@ -33,20 +34,12 @@ static std::shared_ptr createStorageObjectStorage( std::optional format_settings; if (args.storage_def->settings) { - FormatFactorySettings user_format_settings; - - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = context->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.set(change.name, change.value); - } + Settings settings = context->getSettingsCopy(); // Apply changes from SETTINGS clause, with validation. - user_format_settings.applyChanges(args.storage_def->settings->changes); - format_settings = getFormatSettings(context, user_format_settings); + settings.applyChanges(args.storage_def->settings->changes); + + format_settings = getFormatSettings(context, settings); } else { diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h index ea008c2334e..550367b48a2 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index cde41b4afff..462953aaf27 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,10 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsMaxThreads max_parsing_threads; +} namespace ErrorCodes { @@ -436,9 +441,19 @@ Chunk ObjectStorageQueueSource::generateImpl() const auto context = getContext(); reader = StorageObjectStorageSource::createReader( - processor_id, file_iterator, configuration, object_storage, read_from_format_info, - format_settings, nullptr, context, nullptr, log, max_block_size, - context->getSettingsRef().max_parsing_threads.value, /* need_only_count */false); + processor_id, + file_iterator, + configuration, + object_storage, + read_from_format_info, + format_settings, + nullptr, + context, + nullptr, + log, + max_block_size, + context->getSettingsRef()[Setting::max_parsing_threads].value, + /* need_only_count */ false); if (!reader) { diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index b0393a33523..250a0deec4f 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,13 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsString s3queue_default_zookeeper_path; + extern const SettingsBool s3queue_enable_logging_to_s3queue_log; + extern const SettingsBool stream_like_engine_allow_direct_select; + extern const SettingsBool use_concurrency_control; +} namespace ErrorCodes { @@ -44,7 +52,7 @@ namespace { std::string chooseZooKeeperPath(const StorageID & table_id, const Settings & settings, const ObjectStorageQueueSettings & queue_settings) { - std::string zk_path_prefix = settings.s3queue_default_zookeeper_path.value; + std::string zk_path_prefix = settings[Setting::s3queue_default_zookeeper_path].value; if (zk_path_prefix.empty()) zk_path_prefix = "/"; @@ -95,7 +103,7 @@ namespace { case DB::ObjectStorageType::S3: { - if (table_settings.enable_logging_to_queue_log || settings.s3queue_enable_logging_to_s3queue_log) + if (table_settings.enable_logging_to_queue_log || settings[Setting::s3queue_enable_logging_to_s3queue_log]) return context->getS3QueueLog(); return nullptr; } @@ -278,7 +286,7 @@ void StorageObjectStorageQueue::read( size_t max_block_size, size_t) { - if (!local_context->getSettingsRef().stream_like_engine_allow_direct_select) + if (!local_context->getSettingsRef()[Setting::stream_like_engine_allow_direct_select]) { throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "Direct select is not allowed. " "To enable use setting `stream_like_engine_allow_direct_select`"); @@ -477,7 +485,7 @@ bool StorageObjectStorageQueue::streamToViews() block_io.pipeline.complete(std::move(pipe)); block_io.pipeline.setNumThreads(queue_settings->processing_threads_num); - block_io.pipeline.setConcurrencyControl(queue_context->getSettingsRef().use_concurrency_control); + block_io.pipeline.setConcurrencyControl(queue_context->getSettingsRef()[Setting::use_concurrency_control]); std::atomic_size_t rows = 0; block_io.pipeline.setProgressCallback([&](const Progress & progress) { rows += progress.read_rows.load(); }); diff --git a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp index 20968143627..b8e1480862b 100644 --- a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp +++ b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp @@ -1,9 +1,11 @@ #include "config.h" -#include +#include +#include +#include #include #include -#include +#include #if USE_AWS_S3 #include @@ -41,25 +43,10 @@ StoragePtr createQueueStorage(const StorageFactory::Arguments & args) if (args.storage_def->settings) { queue_settings->loadFromQuery(*args.storage_def); - FormatFactorySettings user_format_settings; - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = args.getContext()->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.set(change.name, change.value); - - args.storage_def->settings->changes.removeSetting(change.name); - } - - for (const auto & change : args.storage_def->settings->changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.applyChange(change); - } - format_settings = getFormatSettings(args.getContext(), user_format_settings); + Settings settings = args.getContext()->getSettingsCopy(); + settings.applyChanges(args.storage_def->settings->changes); + format_settings = getFormatSettings(args.getContext(), settings); } else { diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index a9778d5d04d..b798c2475a7 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -39,6 +39,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_materialized_postgresql_table; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 postgresql_connection_attempt_timeout; +} namespace ErrorCodes { @@ -283,7 +289,7 @@ void StorageMaterializedPostgreSQL::read( readFinalFromNestedStorage(query_plan, nested_table, column_names, query_info, context_, processed_stage, max_block_size, num_streams); - auto lock = lockForShare(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + auto lock = lockForShare(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); query_plan.addTableLock(lock); query_plan.addStorageHolder(shared_from_this()); } @@ -574,7 +580,7 @@ void registerStorageMaterializedPostgreSQL(StorageFactory & factory) metadata.setComment(args.comment); if (args.mode <= LoadingStrictnessLevel::CREATE - && !args.getLocalContext()->getSettingsRef().allow_experimental_materialized_postgresql_table) + && !args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_materialized_postgresql_table]) throw Exception(ErrorCodes::BAD_ARGUMENTS, "MaterializedPostgreSQL is an experimental table engine." " You can enable it with the `allow_experimental_materialized_postgresql_table` setting"); @@ -591,9 +597,12 @@ void registerStorageMaterializedPostgreSQL(StorageFactory & factory) auto configuration = StoragePostgreSQL::getConfiguration(args.engine_args, args.getContext()); auto connection_info = postgres::formatConnectionString( - configuration.database, configuration.host, configuration.port, - configuration.username, configuration.password, - args.getContext()->getSettingsRef().postgresql_connection_attempt_timeout); + configuration.database, + configuration.host, + configuration.port, + configuration.username, + configuration.password, + args.getContext()->getSettingsRef()[Setting::postgresql_connection_attempt_timeout]); bool has_settings = args.storage_def->settings; auto postgresql_replication_settings = std::make_unique(); diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.h b/src/Storages/RabbitMQ/RabbitMQSettings.h index f2c0dae21c6..8c13f3dbef1 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.h +++ b/src/Storages/RabbitMQ/RabbitMQSettings.h @@ -1,7 +1,8 @@ #pragma once #include -#include +#include +#include namespace DB { diff --git a/src/Storages/RabbitMQ/RabbitMQSource.cpp b/src/Storages/RabbitMQ/RabbitMQSource.cpp index 15d013245d3..8f3f9daf088 100644 --- a/src/Storages/RabbitMQ/RabbitMQSource.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSource.cpp @@ -1,15 +1,19 @@ #include +#include #include +#include #include #include -#include -#include -#include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsMilliseconds rabbitmq_max_wait_ms; +} static std::pair getHeaders(const StorageSnapshotPtr & storage_snapshot, const Names & column_names) { @@ -138,7 +142,7 @@ Chunk RabbitMQSource::generateImpl() { if (!consumer) { - auto timeout = std::chrono::milliseconds(context->getSettingsRef().rabbitmq_max_wait_ms.totalMilliseconds()); + auto timeout = std::chrono::milliseconds(context->getSettingsRef()[Setting::rabbitmq_max_wait_ms].totalMilliseconds()); consumer = storage.popConsumer(timeout); } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 9e3c40071b5..950cce9861c 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -1,39 +1,47 @@ #include +#include +#include #include #include -#include #include #include #include #include #include -#include #include +#include #include #include -#include -#include #include +#include +#include #include #include +#include #include +#include #include #include -#include -#include #include #include #include #include #include #include +#include #include #include #include -#include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_insert_block_size; + extern const SettingsUInt64 output_format_avro_rows_in_file; + extern const SettingsMilliseconds stream_flush_interval_ms; + extern const SettingsBool stream_like_engine_allow_direct_select; +} static const uint32_t QUEUE_SIZE = 100000; static const auto MAX_FAILED_READ_ATTEMPTS = 10; @@ -389,9 +397,9 @@ void StorageRabbitMQ::deactivateTask(BackgroundSchedulePool::TaskHolder & task, size_t StorageRabbitMQ::getMaxBlockSize() const { - return rabbitmq_settings->rabbitmq_max_block_size.changed - ? rabbitmq_settings->rabbitmq_max_block_size.value - : (getContext()->getSettingsRef().max_insert_block_size.value / num_consumers); + return rabbitmq_settings->rabbitmq_max_block_size.changed + ? rabbitmq_settings->rabbitmq_max_block_size.value + : (getContext()->getSettingsRef()[Setting::max_insert_block_size].value / num_consumers); } @@ -743,7 +751,7 @@ void StorageRabbitMQ::read( return; } - if (!local_context->getSettingsRef().stream_like_engine_allow_direct_select) + if (!local_context->getSettingsRef()[Setting::stream_like_engine_allow_direct_select]) throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "Direct select is not allowed. To enable use setting `stream_like_engine_allow_direct_select`"); @@ -768,7 +776,7 @@ void StorageRabbitMQ::read( uint64_t max_execution_time_ms = rabbitmq_settings->rabbitmq_flush_interval_ms.changed ? rabbitmq_settings->rabbitmq_flush_interval_ms - : static_cast(getContext()->getSettingsRef().stream_flush_interval_ms.totalMilliseconds()); + : static_cast(getContext()->getSettingsRef()[Setting::stream_flush_interval_ms].totalMilliseconds()); for (size_t i = 0; i < num_created_consumers; ++i) { @@ -815,8 +823,8 @@ SinkToStoragePtr StorageRabbitMQ::write(const ASTPtr &, const StorageMetadataPtr configuration, routing_keys, exchange_name, exchange_type, producer_id.fetch_add(1), persistent, shutdown_called, log); size_t max_rows = max_rows_per_message; /// Need for backward compatibility. - if (format_name == "Avro" && local_context->getSettingsRef().output_format_avro_rows_in_file.changed) - max_rows = local_context->getSettingsRef().output_format_avro_rows_in_file.value; + if (format_name == "Avro" && local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].changed) + max_rows = local_context->getSettingsRef()[Setting::output_format_avro_rows_in_file].value; return std::make_shared( metadata_snapshot->getSampleBlockNonMaterialized(), getFormatName(), @@ -1105,7 +1113,7 @@ bool StorageRabbitMQ::tryStreamToViews() uint64_t max_execution_time_ms = rabbitmq_settings->rabbitmq_flush_interval_ms.changed ? rabbitmq_settings->rabbitmq_flush_interval_ms - : static_cast(getContext()->getSettingsRef().stream_flush_interval_ms.totalMilliseconds()); + : static_cast(getContext()->getSettingsRef()[Setting::stream_flush_interval_ms].totalMilliseconds()); for (size_t i = 0; i < num_created_consumers; ++i) { diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp index add12413853..6674202fe81 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp +++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp @@ -19,6 +19,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} bool needRewriteQueryWithFinalForStorage(const Names & column_names, const StoragePtr & storage) { @@ -39,7 +43,7 @@ void readFinalFromNestedStorage( size_t num_streams) { NameSet column_names_set = NameSet(column_names.begin(), column_names.end()); - auto lock = nested_storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout); + auto lock = nested_storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); const auto & nested_metadata = nested_storage->getInMemoryMetadataPtr(); Block nested_header = nested_metadata->getSampleBlock(); diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 9c8c4c2fe79..3b6c596b159 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -15,6 +15,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_respect_aliases; +} namespace ErrorCodes { @@ -264,7 +268,7 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder( /// Currently we only support alias column without any function wrapper, /// i.e.: `order by aliased_column` can have this optimization, but `order by function(aliased_column)` can not. /// This suits most cases. - if (context->getSettingsRef().optimize_respect_aliases && !aliased_columns.empty()) + if (context->getSettingsRef()[Setting::optimize_respect_aliases] && !aliased_columns.empty()) { SortDescription aliases_sort_description = required_sort_description; ManyExpressionActions aliases_actions = elements_actions; diff --git a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp index f6127f3ed8a..3dfeee8d912 100644 --- a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp +++ b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp @@ -30,6 +30,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 min_insert_block_size_rows; +} namespace ErrorCodes { @@ -86,7 +90,8 @@ EmbeddedRocksDBBulkSink::EmbeddedRocksDBBulkSink( } serializations = getHeader().getSerializations(); - min_block_size_rows = std::max(storage.getSettings().bulk_insert_block_size, getContext()->getSettingsRef().min_insert_block_size_rows); + min_block_size_rows + = std::max(storage.getSettings().bulk_insert_block_size, getContext()->getSettingsRef()[Setting::min_insert_block_size_rows]); /// If max_insert_threads > 1 we may have multiple EmbeddedRocksDBBulkSink and getContext()->getCurrentQueryId() is not guarantee to /// to have a distinct path. Also we cannot use query id as directory name here, because it could be defined by user and not suitable diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 50f6266cb2f..617e9331795 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -52,6 +52,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool optimize_trivial_approximate_count_query; +} namespace ErrorCodes { @@ -812,7 +816,7 @@ Chunk StorageEmbeddedRocksDB::getBySerializedKeys( std::optional StorageEmbeddedRocksDB::totalRows(const Settings & query_settings) const { - if (!query_settings.optimize_trivial_approximate_count_query) + if (!query_settings[Setting::optimize_trivial_approximate_count_query]) return {}; std::shared_lock lock(rocksdb_ptr_mx); if (!rocksdb_ptr) diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index ea5852de830..632254cf5ce 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -17,18 +17,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool system_events_show_zero_values; +} namespace ErrorCodes { extern const int LOGICAL_ERROR; } -} - -namespace DB -{ - - ColumnsDescription StorageSystemRocksDB::getColumnsDescription() { return ColumnsDescription @@ -106,7 +104,7 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con col_table_to_filter = filtered_block.getByName("table").column; } - bool show_zeros = context->getSettingsRef().system_events_show_zero_values; + bool show_zeros = context->getSettingsRef()[Setting::system_events_show_zero_values]; for (size_t i = 0, tables_size = col_database_to_filter->size(); i < tables_size; ++i) { String database = (*col_database_to_filter)[i].safeGet(); diff --git a/src/Storages/SetSettings.h b/src/Storages/SetSettings.h index 98c1f3d97f5..bd14859ff1e 100644 --- a/src/Storages/SetSettings.h +++ b/src/Storages/SetSettings.h @@ -1,8 +1,8 @@ #pragma once #include -#include - +#include +#include namespace DB { diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index f753d369d2d..fc1126680e9 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -71,6 +71,13 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool insert_allow_materialized_columns; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 readonly; +} namespace ErrorCodes { @@ -251,7 +258,7 @@ void StorageBuffer::read( size_t max_block_size, size_t num_streams) { - bool allow_experimental_analyzer = local_context->getSettingsRef().allow_experimental_analyzer; + bool allow_experimental_analyzer = local_context->getSettingsRef()[Setting::allow_experimental_analyzer]; if (allow_experimental_analyzer && processed_stage > QueryProcessingStage::FetchColumns) { @@ -278,7 +285,8 @@ void StorageBuffer::read( if (auto destination = getDestinationTable()) { - auto destination_lock = destination->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto destination_lock + = destination->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto destination_metadata_snapshot = destination->getInMemoryMetadataPtr(); auto destination_snapshot = destination->getStorageSnapshot(destination_metadata_snapshot, local_context); @@ -761,7 +769,7 @@ SinkToStoragePtr StorageBuffer::write(const ASTPtr & /*query*/, const StorageMet void StorageBuffer::startup() { - if (getContext()->getSettingsRef().readonly) + if (getContext()->getSettingsRef()[Setting::readonly]) { LOG_WARNING(log, "Storage {} is run with readonly settings, it will not be able to insert data. Set appropriate buffer_profile to fix this.", getName()); } @@ -1264,7 +1272,7 @@ void registerStorageBuffer(StorageFactory & factory) max, flush, destination_id, - static_cast(args.getLocalContext()->getSettingsRef().insert_allow_materialized_columns)); + static_cast(args.getLocalContext()->getSettingsRef()[Setting::insert_allow_materialized_columns])); }, { .supports_parallel_insert = true, diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp index b43cc4fa426..4a290ca3a94 100644 --- a/src/Storages/StorageDictionary.cpp +++ b/src/Storages/StorageDictionary.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool dictionary_validate_primary_key_type; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -112,7 +117,12 @@ StorageDictionary::StorageDictionary( Location location_, ContextPtr context_) : StorageDictionary( - table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_, context_->getSettingsRef().dictionary_validate_primary_key_type)}, comment, location_, context_) + table_id_, + dictionary_name_, + ColumnsDescription{getNamesAndTypes(dictionary_structure_, context_->getSettingsRef()[Setting::dictionary_validate_primary_key_type])}, + comment, + location_, + context_) { } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 0b80858800b..438c31b5cc3 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -131,6 +131,33 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_nondeterministic_optimize_skip_unused_shards; + extern const SettingsBool async_socket_for_remote; + extern const SettingsBool async_query_sending_for_remote; + extern const SettingsBool distributed_background_insert_batch; + extern const SettingsUInt64 distributed_background_insert_timeout; + extern const SettingsMilliseconds distributed_background_insert_sleep_time_ms; + extern const SettingsMilliseconds distributed_background_insert_max_sleep_time_ms; + extern const SettingsBool distributed_background_insert_split_batch_on_failure; + extern const SettingsUInt64 distributed_group_by_no_merge; + extern const SettingsBool distributed_foreground_insert; + extern const SettingsUInt64 distributed_push_down_limit; + extern const SettingsBool extremes; + extern const SettingsUInt64 force_optimize_skip_unused_shards; + extern const SettingsBool insert_allow_materialized_columns; + extern const SettingsBool insert_distributed_one_random_shard; + extern const SettingsUInt64 insert_shard_id; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_distributed_depth; + extern const SettingsNonZeroUInt64 max_parallel_replicas; + extern const SettingsBool optimize_distributed_group_by_sharding_key; + extern const SettingsBool optimize_skip_unused_shards; + extern const SettingsUInt64 optimize_skip_unused_shards_limit; + extern const SettingsUInt64 parallel_distributed_insert_select; +} namespace ErrorCodes { @@ -271,7 +298,7 @@ size_t getClusterQueriedNodes(const Settings & settings, const ClusterPtr & clus { size_t num_local_shards = cluster->getLocalShardCount(); size_t num_remote_shards = cluster->getRemoteShardCount(); - return (num_remote_shards + num_local_shards) * settings.max_parallel_replicas; + return (num_remote_shards + num_local_shards) * settings[Setting::max_parallel_replicas]; } } @@ -394,7 +421,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( if (!local_context->canUseParallelReplicasCustomKeyForCluster(*cluster)) { - if (nodes > 1 && settings.optimize_skip_unused_shards) + if (nodes > 1 && settings[Setting::optimize_skip_unused_shards]) { /// Always calculate optimized cluster here, to avoid conditions during read() /// (Anyway it will be calculated in the read()) @@ -418,11 +445,11 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( } } - if (settings.distributed_group_by_no_merge) + if (settings[Setting::distributed_group_by_no_merge]) { - if (settings.distributed_group_by_no_merge == DISTRIBUTED_GROUP_BY_NO_MERGE_AFTER_AGGREGATION) + if (settings[Setting::distributed_group_by_no_merge] == DISTRIBUTED_GROUP_BY_NO_MERGE_AFTER_AGGREGATION) { - if (settings.distributed_push_down_limit) + if (settings[Setting::distributed_push_down_limit]) return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; else return QueryProcessingStage::WithMergeableStateAfterAggregation; @@ -478,14 +505,11 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( std::optional StorageDistributed::getOptimizedQueryProcessingStageAnalyzer(const SelectQueryInfo & query_info, const Settings & settings) const { - bool optimize_sharding_key_aggregation = - settings.optimize_skip_unused_shards && - settings.optimize_distributed_group_by_sharding_key && - has_sharding_key && - (settings.allow_nondeterministic_optimize_skip_unused_shards || sharding_key_is_deterministic); + bool optimize_sharding_key_aggregation = settings[Setting::optimize_skip_unused_shards] && settings[Setting::optimize_distributed_group_by_sharding_key] + && has_sharding_key && (settings[Setting::allow_nondeterministic_optimize_skip_unused_shards] || sharding_key_is_deterministic); QueryProcessingStage::Enum default_stage = QueryProcessingStage::WithMergeableStateAfterAggregation; - if (settings.distributed_push_down_limit) + if (settings[Setting::distributed_push_down_limit]) default_stage = QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; const auto & query_node = query_info.query_tree->as(); @@ -526,7 +550,7 @@ std::optional StorageDistributed::getOptimizedQueryP if (hasWindowFunctionNodes(query_info.query_tree)) return {}; // TODO: extremes support can be implemented - if (settings.extremes) + if (settings[Setting::extremes]) return {}; // DISTINCT @@ -565,14 +589,11 @@ std::optional StorageDistributed::getOptimizedQueryP std::optional StorageDistributed::getOptimizedQueryProcessingStage(const SelectQueryInfo & query_info, const Settings & settings) const { - bool optimize_sharding_key_aggregation = - settings.optimize_skip_unused_shards && - settings.optimize_distributed_group_by_sharding_key && - has_sharding_key && - (settings.allow_nondeterministic_optimize_skip_unused_shards || sharding_key_is_deterministic); + bool optimize_sharding_key_aggregation = settings[Setting::optimize_skip_unused_shards] && settings[Setting::optimize_distributed_group_by_sharding_key] + && has_sharding_key && (settings[Setting::allow_nondeterministic_optimize_skip_unused_shards] || sharding_key_is_deterministic); QueryProcessingStage::Enum default_stage = QueryProcessingStage::WithMergeableStateAfterAggregation; - if (settings.distributed_push_down_limit) + if (settings[Setting::distributed_push_down_limit]) default_stage = QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; const auto & select = query_info.query->as(); @@ -606,7 +627,7 @@ std::optional StorageDistributed::getOptimizedQueryP if (query_info.has_window) return {}; // TODO: extremes support can be implemented - if (settings.extremes) + if (settings[Setting::extremes]) return {}; // DISTINCT @@ -807,7 +828,7 @@ void StorageDistributed::read( const auto & settings = local_context->getSettingsRef(); - if (settings.allow_experimental_analyzer) + if (settings[Setting::allow_experimental_analyzer]) { StorageID remote_storage_id = StorageID::createEmpty(); if (!remote_table_function_ptr) @@ -891,23 +912,23 @@ SinkToStoragePtr StorageDistributed::write(const ASTPtr &, const StorageMetadata auto shard_num = cluster->getLocalShardCount() + cluster->getRemoteShardCount(); /// If sharding key is not specified, then you can only write to a shard containing only one shard - if (!settings.insert_shard_id && !settings.insert_distributed_one_random_shard && !has_sharding_key && shard_num >= 2) + if (!settings[Setting::insert_shard_id] && !settings[Setting::insert_distributed_one_random_shard] && !has_sharding_key && shard_num >= 2) { throw Exception(ErrorCodes::STORAGE_REQUIRES_PARAMETER, "Method write is not supported by storage {} with more than one shard and no sharding key provided", getName()); } - if (settings.insert_shard_id && settings.insert_shard_id > shard_num) + if (settings[Setting::insert_shard_id] && settings[Setting::insert_shard_id] > shard_num) { throw Exception(ErrorCodes::INVALID_SHARD_ID, "Shard id should be range from 1 to shard number"); } /// Force sync insertion if it is remote() table function - bool insert_sync = settings.distributed_foreground_insert || settings.insert_shard_id || owned_cluster; - auto timeout = settings.distributed_background_insert_timeout; + bool insert_sync = settings[Setting::distributed_foreground_insert] || settings[Setting::insert_shard_id] || owned_cluster; + auto timeout = settings[Setting::distributed_background_insert_timeout]; Names columns_to_send; - if (settings.insert_allow_materialized_columns) + if (settings[Setting::insert_allow_materialized_columns]) columns_to_send = metadata_snapshot->getSampleBlock().getNames(); else columns_to_send = metadata_snapshot->getSampleBlockNonMaterialized().getNames(); @@ -970,7 +991,7 @@ std::optional StorageDistributed::distributedWriteBetweenDistribu return {}; } - if (settings.parallel_distributed_insert_select == PARALLEL_DISTRIBUTED_INSERT_SELECT_ALL) + if (settings[Setting::parallel_distributed_insert_select] == PARALLEL_DISTRIBUTED_INSERT_SELECT_ALL) { new_query->table_id = StorageID(getRemoteDatabaseName(), getRemoteTableName()); /// Reset table function for INSERT INTO remote()/cluster() @@ -1016,7 +1037,8 @@ std::optional StorageDistributed::distributedWriteBetweenDistribu /// INSERT SELECT query returns empty block auto remote_query_executor = std::make_shared(std::move(connections), new_query_str, Block{}, query_context); - QueryPipeline remote_pipeline(std::make_shared(remote_query_executor, false, settings.async_socket_for_remote, settings.async_query_sending_for_remote)); + QueryPipeline remote_pipeline(std::make_shared( + remote_query_executor, false, settings[Setting::async_socket_for_remote], settings[Setting::async_query_sending_for_remote])); remote_pipeline.complete(std::make_shared(remote_query_executor->getHeader())); pipeline.addCompletedPipeline(std::move(remote_pipeline)); @@ -1031,7 +1053,7 @@ static std::optional getFilterFromQuery(const ASTPtr & ast, ContextP QueryPlan plan; SelectQueryOptions options; options.only_analyze = true; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(ast, context, options); plan = std::move(interpreter).extractQueryPlan(); @@ -1091,7 +1113,7 @@ std::optional StorageDistributed::distributedWriteFromClusterStor auto dst_cluster = getCluster(); auto new_query = std::dynamic_pointer_cast(query.clone()); - if (settings.parallel_distributed_insert_select == PARALLEL_DISTRIBUTED_INSERT_SELECT_ALL) + if (settings[Setting::parallel_distributed_insert_select] == PARALLEL_DISTRIBUTED_INSERT_SELECT_ALL) { new_query->table_id = StorageID(getRemoteDatabaseName(), getRemoteTableName()); /// Reset table function for INSERT INTO remote()/cluster() @@ -1134,7 +1156,8 @@ std::optional StorageDistributed::distributedWriteFromClusterStor QueryProcessingStage::Complete, extension); - QueryPipeline remote_pipeline(std::make_shared(remote_query_executor, false, settings.async_socket_for_remote, settings.async_query_sending_for_remote)); + QueryPipeline remote_pipeline(std::make_shared( + remote_query_executor, false, settings[Setting::async_socket_for_remote], settings[Setting::async_query_sending_for_remote])); remote_pipeline.complete(std::make_shared(remote_query_executor->getHeader())); pipeline.addCompletedPipeline(std::move(remote_pipeline)); @@ -1148,7 +1171,7 @@ std::optional StorageDistributed::distributedWriteFromClusterStor std::optional StorageDistributed::distributedWrite(const ASTInsertQuery & query, ContextPtr local_context) { const Settings & settings = local_context->getSettingsRef(); - if (settings.max_distributed_depth && local_context->getClientInfo().distributed_depth >= settings.max_distributed_depth) + if (settings[Setting::max_distributed_depth] && local_context->getClientInfo().distributed_depth >= settings[Setting::max_distributed_depth]) throw Exception(ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH, "Maximum distributed depth exceeded"); auto & select = query.select->as(); @@ -1486,7 +1509,7 @@ ClusterPtr StorageDistributed::getOptimizedCluster( ClusterPtr cluster = getCluster(); const Settings & settings = local_context->getSettingsRef(); - bool sharding_key_is_usable = settings.allow_nondeterministic_optimize_skip_unused_shards || sharding_key_is_deterministic; + bool sharding_key_is_usable = settings[Setting::allow_nondeterministic_optimize_skip_unused_shards] || sharding_key_is_deterministic; if (has_sharding_key && sharding_key_is_usable) { @@ -1495,7 +1518,7 @@ ClusterPtr StorageDistributed::getOptimizedCluster( return optimized; } - UInt64 force = settings.force_optimize_skip_unused_shards; + UInt64 force = settings[Setting::force_optimize_skip_unused_shards]; if (force == FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_ALWAYS || (force == FORCE_OPTIMIZE_SKIP_UNUSED_SHARDS_HAS_SHARDING_KEY && has_sharding_key)) { if (!has_sharding_key) @@ -1545,7 +1568,7 @@ ClusterPtr StorageDistributed::skipUnusedShardsWithAnalyzer( if (!query_info.filter_actions_dag) return nullptr; - size_t limit = local_context->getSettingsRef().optimize_skip_unused_shards_limit; + size_t limit = local_context->getSettingsRef()[Setting::optimize_skip_unused_shards_limit]; if (!limit || limit > SSIZE_MAX) { throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "optimize_skip_unused_shards_limit out of range (0, {}]", SSIZE_MAX); @@ -1585,7 +1608,7 @@ ClusterPtr StorageDistributed::skipUnusedShards( const StorageSnapshotPtr & storage_snapshot, ContextPtr local_context) const { - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) return skipUnusedShardsWithAnalyzer(cluster, query_info, storage_snapshot, local_context); const auto & select = query_info.query->as(); @@ -1616,7 +1639,7 @@ ClusterPtr StorageDistributed::skipUnusedShards( replaceConstantExpressions(condition_ast, local_context, storage_snapshot->metadata->getColumns().getAll(), shared_from_this(), storage_snapshot); - size_t limit = local_context->getSettingsRef().optimize_skip_unused_shards_limit; + size_t limit = local_context->getSettingsRef()[Setting::optimize_skip_unused_shards_limit]; if (!limit || limit > SSIZE_MAX) { throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "optimize_skip_unused_shards_limit out of range (0, {}]", SSIZE_MAX); @@ -1627,10 +1650,11 @@ ClusterPtr StorageDistributed::skipUnusedShards( if (!limit) { - LOG_DEBUG(log, + LOG_DEBUG( + log, "Number of values for sharding key exceeds optimize_skip_unused_shards_limit={}, " "try to increase it, but note that this may increase query processing time.", - local_context->getSettingsRef().optimize_skip_unused_shards_limit); + local_context->getSettingsRef()[Setting::optimize_skip_unused_shards_limit]); return nullptr; } @@ -1681,7 +1705,7 @@ void StorageDistributed::flushClusterNodesAllData(ContextPtr local_context, cons void StorageDistributed::flushClusterNodesAllDataImpl(ContextPtr local_context, const SettingsChanges & settings_changes, bool flush) { /// Sync SYSTEM FLUSH DISTRIBUTED with TRUNCATE - auto table_lock = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto table_lock = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); std::vector> directory_queues; @@ -1907,13 +1931,15 @@ void registerStorageDistributed(StorageFactory & factory) /// Set default values from the distributed_background_insert_* global context settings. if (!distributed_settings.background_insert_batch.changed) - distributed_settings.background_insert_batch = context->getSettingsRef().distributed_background_insert_batch; + distributed_settings.background_insert_batch = context->getSettingsRef()[Setting::distributed_background_insert_batch]; if (!distributed_settings.background_insert_split_batch_on_failure.changed) - distributed_settings.background_insert_split_batch_on_failure = context->getSettingsRef().distributed_background_insert_split_batch_on_failure; + distributed_settings.background_insert_split_batch_on_failure + = context->getSettingsRef()[Setting::distributed_background_insert_split_batch_on_failure]; if (!distributed_settings.background_insert_sleep_time_ms.changed) - distributed_settings.background_insert_sleep_time_ms = context->getSettingsRef().distributed_background_insert_sleep_time_ms; + distributed_settings.background_insert_sleep_time_ms = context->getSettingsRef()[Setting::distributed_background_insert_sleep_time_ms]; if (!distributed_settings.background_insert_max_sleep_time_ms.changed) - distributed_settings.background_insert_max_sleep_time_ms = context->getSettingsRef().distributed_background_insert_max_sleep_time_ms; + distributed_settings.background_insert_max_sleep_time_ms + = context->getSettingsRef()[Setting::distributed_background_insert_max_sleep_time_ms]; return std::make_shared( args.table_id, diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index 0094723e3fd..dd1b70364e2 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -29,6 +29,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -150,7 +155,7 @@ void StorageExecutable::read( for (auto & input_query : input_queries) { QueryPipelineBuilder builder; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) builder = InterpreterSelectQueryAnalyzer(input_query, context, {}).buildQueryPipeline(); else builder = InterpreterSelectWithUnionQuery(input_query, context, {}).buildQueryPipeline(); @@ -228,7 +233,7 @@ void registerStorageExecutable(StorageFactory & factory) { size_t max_command_execution_time = 10; - size_t max_execution_time_seconds = static_cast(args.getContext()->getSettingsRef().max_execution_time.totalSeconds()); + size_t max_execution_time_seconds = static_cast(args.getContext()->getSettingsRef()[Setting::max_execution_time].totalSeconds()); if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 9fc8b588c89..450f4384af4 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -24,6 +24,15 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 glob_expansion_max_elements; + extern const SettingsUInt64 postgresql_connection_attempt_timeout; + extern const SettingsBool postgresql_connection_pool_auto_close_connection; + extern const SettingsUInt64 postgresql_connection_pool_retries; + extern const SettingsUInt64 postgresql_connection_pool_size; + extern const SettingsUInt64 postgresql_connection_pool_wait_timeout; +} namespace ErrorCodes { @@ -105,7 +114,7 @@ void registerStorageExternalDistributed(StorageFactory & factory) auto context = args.getLocalContext(); const auto & settings = context->getSettingsRef(); - size_t max_addresses = settings.glob_expansion_max_elements; + size_t max_addresses = settings[Setting::glob_expansion_max_elements]; auto get_addresses = [&](const std::string addresses_expr) { return parseRemoteDescription(addresses_expr, 0, addresses_expr.size(), ',', max_addresses); @@ -190,11 +199,11 @@ void registerStorageExternalDistributed(StorageFactory & factory) configuration.addresses = parseRemoteDescriptionForExternalDatabase(shard_address, max_addresses, 5432); auto pool = std::make_shared( configuration, - settings.postgresql_connection_pool_size, - settings.postgresql_connection_pool_wait_timeout, - settings.postgresql_connection_pool_retries, - settings.postgresql_connection_pool_auto_close_connection, - settings.postgresql_connection_attempt_timeout); + settings[Setting::postgresql_connection_pool_size], + settings[Setting::postgresql_connection_pool_wait_timeout], + settings[Setting::postgresql_connection_pool_retries], + settings[Setting::postgresql_connection_pool_auto_close_connection], + settings[Setting::postgresql_connection_attempt_timeout]); shards.insert(std::make_shared( args.table_id, std::move(pool), configuration.table, args.columns, args.constraints, String{}, context)); } diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index b95ccedb093..fe7ef5aed54 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -10,6 +10,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -232,7 +236,7 @@ StoragePtr StorageFactory::get( storage_def->engine->arguments->children = empty_engine_args; } - if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries) + if (local_context->hasQueryContext() && local_context->getSettingsRef()[Setting::log_queries]) local_context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Storage, name); return res; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 55bc8083ec8..1d846b6bb0f 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -56,6 +55,7 @@ #include #include "base/defines.h" +#include #include #include @@ -79,6 +79,27 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_archive_path_syntax; + extern const SettingsBool engine_file_allow_create_multiple_files; + extern const SettingsBool engine_file_empty_if_not_exists; + extern const SettingsBool engine_file_skip_empty_files; + extern const SettingsBool engine_file_truncate_on_insert; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsSeconds max_execution_time; + extern const SettingsMaxThreads max_parsing_threads; + extern const SettingsUInt64 max_read_buffer_size; + extern const SettingsBool optimize_count_from_files; + extern const SettingsUInt64 output_format_compression_level; + extern const SettingsUInt64 output_format_compression_zstd_window_log; + extern const SettingsBool parallelize_output_from_storages; + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsBool schema_inference_use_cache_for_file; + extern const SettingsLocalFSReadMethod storage_file_read_method; + extern const SettingsBool use_cache_for_count_from_files; + extern const SettingsInt64 zstd_window_log_max; +} namespace ErrorCodes { @@ -242,7 +263,7 @@ std::unique_ptr selectReadBuffer( const struct stat & file_stat, ContextPtr context) { - auto read_method = context->getSettingsRef().storage_file_read_method; + auto read_method = context->getSettingsRef()[Setting::storage_file_read_method]; /** Using mmap on server-side is unsafe for the following reasons: * - concurrent modifications of a file will result in SIGBUS; @@ -280,9 +301,9 @@ std::unique_ptr selectReadBuffer( if (S_ISREG(file_stat.st_mode) && (read_method == LocalFSReadMethod::pread || read_method == LocalFSReadMethod::mmap)) { if (use_table_fd) - res = std::make_unique(table_fd, context->getSettingsRef().max_read_buffer_size); + res = std::make_unique(table_fd, context->getSettingsRef()[Setting::max_read_buffer_size]); else - res = std::make_unique(current_path, context->getSettingsRef().max_read_buffer_size); + res = std::make_unique(current_path, context->getSettingsRef()[Setting::max_read_buffer_size]); ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary); } @@ -291,10 +312,7 @@ std::unique_ptr selectReadBuffer( #if USE_LIBURING auto & reader = getIOUringReaderOrThrow(context); res = std::make_unique( - reader, - Priority{}, - current_path, - context->getSettingsRef().max_read_buffer_size); + reader, Priority{}, current_path, context->getSettingsRef()[Setting::max_read_buffer_size]); #else throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Read method io_uring is only supported in Linux"); #endif @@ -302,9 +320,9 @@ std::unique_ptr selectReadBuffer( else { if (use_table_fd) - res = std::make_unique(table_fd, context->getSettingsRef().max_read_buffer_size); + res = std::make_unique(table_fd, context->getSettingsRef()[Setting::max_read_buffer_size]); else - res = std::make_unique(current_path, context->getSettingsRef().max_read_buffer_size); + res = std::make_unique(current_path, context->getSettingsRef()[Setting::max_read_buffer_size]); ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary); } @@ -346,7 +364,7 @@ std::unique_ptr createReadBuffer( std::unique_ptr nested_buffer = selectReadBuffer(current_path, use_table_fd, table_fd, file_stat, context); - int zstd_window_log_max = static_cast(context->getSettingsRef().zstd_window_log_max); + int zstd_window_log_max = static_cast(context->getSettingsRef()[Setting::zstd_window_log_max]); return wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method, zstd_window_log_max); } @@ -443,7 +461,7 @@ namespace /// For default mode check cached columns for all paths on first iteration. /// If we have cached columns, next() won't be called again. - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::DEFAULT) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::DEFAULT) { if (auto cached_columns = tryGetColumnsFromCache(paths)) return {nullptr, cached_columns, format}; @@ -474,10 +492,10 @@ namespace path = paths[current_index++]; file_stat = getFileStat(path, false, -1, "File"); - } while (getContext()->getSettingsRef().engine_file_skip_empty_files && file_stat.st_size == 0); + } while (getContext()->getSettingsRef()[Setting::engine_file_skip_empty_files] && file_stat.st_size == 0); /// For union mode, check cached columns only for current path, because schema can be different for different files. - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::UNION) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::UNION) { if (auto cached_columns = tryGetColumnsFromCache({path})) return {nullptr, cached_columns, format}; @@ -488,7 +506,7 @@ namespace void setNumRowsToLastFile(size_t num_rows) override { - if (!getContext()->getSettingsRef().use_cache_for_count_from_files) + if (!getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) return; auto key = getKeyForSchemaCache(paths[current_index - 1], *format, format_settings, getContext()); @@ -497,8 +515,8 @@ namespace void setSchemaToLastFile(const ColumnsDescription & columns) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_file - || getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::UNION) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_file] + || getContext()->getSettingsRef()[Setting::schema_inference_mode] != SchemaInferenceMode::UNION) return; /// For union mode, schema can be different for different files, so we need to @@ -509,8 +527,8 @@ namespace void setResultingSchema(const ColumnsDescription & columns) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_file - || getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::DEFAULT) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_file] + || getContext()->getSettingsRef()[Setting::schema_inference_mode] != SchemaInferenceMode::DEFAULT) return; /// For default mode we cache resulting schema for all paths. @@ -544,7 +562,7 @@ namespace std::optional tryGetColumnsFromCache(const Strings & paths_) { auto context = getContext(); - if (!context->getSettingsRef().schema_inference_use_cache_for_file) + if (!context->getSettingsRef()[Setting::schema_inference_use_cache_for_file]) return std::nullopt; /// Check if the cache contains one of the paths. @@ -614,7 +632,7 @@ namespace { /// For default mode check cached columns for all initial archive paths (maybe with globs) on first iteration. /// If we have cached columns, next() won't be called again. - if (is_first && getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::DEFAULT) + if (is_first && getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::DEFAULT) { for (const auto & archive : archive_info.paths_to_archives) { @@ -649,7 +667,7 @@ namespace file_stat = getFileStat(archive, false, -1, "File"); if (file_stat.st_size == 0) { - if (getContext()->getSettingsRef().engine_file_skip_empty_files) + if (getContext()->getSettingsRef()[Setting::engine_file_skip_empty_files]) { ++current_archive_index; continue; @@ -732,7 +750,7 @@ namespace { /// For union mode next() will be called again even if we found cached columns, /// so we need to remember last_read_buffer to continue iterating through files in archive. - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::UNION) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::UNION) last_read_buffer = archive_reader->readFile(std::move(file_enumerator)); return {nullptr, cached_schema, format}; } @@ -754,7 +772,7 @@ namespace void setNumRowsToLastFile(size_t num_rows) override { - if (!getContext()->getSettingsRef().use_cache_for_count_from_files) + if (!getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) return; auto key = getKeyForSchemaCache(last_read_file_path, *format, format_settings, getContext()); @@ -763,8 +781,8 @@ namespace void setSchemaToLastFile(const ColumnsDescription & columns) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_file - || getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::UNION) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_file] + || getContext()->getSettingsRef()[Setting::schema_inference_mode] != SchemaInferenceMode::UNION) return; /// For union mode, schema can be different for different files in archive, so we need to @@ -776,8 +794,8 @@ namespace void setResultingSchema(const ColumnsDescription & columns) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_file - || getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::DEFAULT) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_file] + || getContext()->getSettingsRef()[Setting::schema_inference_mode] != SchemaInferenceMode::DEFAULT) return; /// For default mode we cache resulting schema for all paths. @@ -824,7 +842,7 @@ namespace std::optional tryGetSchemaFromCache(const std::string & archive_path, const std::string & full_path) { auto context = getContext(); - if (!context->getSettingsRef().schema_inference_use_cache_for_file) + if (!context->getSettingsRef()[Setting::schema_inference_use_cache_for_file]) return std::nullopt; struct stat file_stat; @@ -1122,9 +1140,9 @@ void StorageFile::setStorageMetadata(CommonArguments args) static std::chrono::seconds getLockTimeout(const ContextPtr & context) { const Settings & settings = context->getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); + Int64 lock_timeout = settings[Setting::lock_acquire_timeout].totalSeconds(); + if (settings[Setting::max_execution_time].totalSeconds() != 0 && settings[Setting::max_execution_time].totalSeconds() < lock_timeout) + lock_timeout = settings[Setting::max_execution_time].totalSeconds(); return std::chrono::seconds{lock_timeout}; } @@ -1265,7 +1283,7 @@ void StorageFileSource::setKeyCondition(const std::optional & filter bool StorageFileSource::tryGetCountFromCache(const struct stat & file_stat) { - if (!getContext()->getSettingsRef().use_cache_for_count_from_files) + if (!getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) return false; auto num_rows_from_cache = tryGetNumRowsFromCache(current_path, file_stat.st_mtime); @@ -1307,7 +1325,7 @@ Chunk StorageFileSource::generate() return {}; auto file_stat = getFileStat(archive, storage->use_table_fd, storage->table_fd, storage->getName()); - if (getContext()->getSettingsRef().engine_file_skip_empty_files && file_stat.st_size == 0) + if (getContext()->getSettingsRef()[Setting::engine_file_skip_empty_files] && file_stat.st_size == 0) continue; archive_reader = createArchiveReader(archive); @@ -1335,7 +1353,7 @@ Chunk StorageFileSource::generate() return {}; current_archive_stat = getFileStat(archive, storage->use_table_fd, storage->table_fd, storage->getName()); - if (getContext()->getSettingsRef().engine_file_skip_empty_files && current_archive_stat.st_size == 0) + if (getContext()->getSettingsRef()[Setting::engine_file_skip_empty_files] && current_archive_stat.st_size == 0) continue; archive_reader = createArchiveReader(archive); @@ -1397,7 +1415,7 @@ Chunk StorageFileSource::generate() current_file_size = file_stat.st_size; current_file_last_modified = Poco::Timestamp::fromEpochTime(file_stat.st_mtime); - if (getContext()->getSettingsRef().engine_file_skip_empty_files && file_stat.st_size == 0) + if (getContext()->getSettingsRef()[Setting::engine_file_skip_empty_files] && file_stat.st_size == 0) continue; if (need_only_count && tryGetCountFromCache(file_stat)) @@ -1416,7 +1434,7 @@ Chunk StorageFileSource::generate() chassert(file_num > 0); - const auto max_parsing_threads = std::max(settings.max_parsing_threads / file_num, 1UL); + const auto max_parsing_threads = std::max(settings[Setting::max_parsing_threads] / file_num, 1UL); input_format = FormatFactory::instance().getInput( storage->format_name, *read_buf, block_for_format, getContext(), max_block_size, storage->format_settings, max_parsing_threads, std::nullopt, /*is_remote_fs*/ false, CompressionMethod::None, need_only_count); @@ -1478,7 +1496,7 @@ Chunk StorageFileSource::generate() if (storage->use_table_fd) finished_generate = true; - if (input_format && storage->format_name != "Distributed" && getContext()->getSettingsRef().use_cache_for_count_from_files) + if (input_format && storage->format_name != "Distributed" && getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) addNumRowsToCache(current_path, total_rows_in_file); total_rows_in_file = 0; @@ -1601,7 +1619,7 @@ void StorageFile::read( if (p->size() == 1 && !fs::exists(p->at(0))) { - if (!context->getSettingsRef().engine_file_empty_if_not_exists) + if (!context->getSettingsRef()[Setting::engine_file_empty_if_not_exists]) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "File {} doesn't exist", p->at(0)); auto header = storage_snapshot->getSampleBlockForColumns(column_names); @@ -1614,7 +1632,7 @@ void StorageFile::read( auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(context)); bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) - && context->getSettingsRef().optimize_count_from_files; + && context->getSettingsRef()[Setting::optimize_count_from_files]; auto reading = std::make_unique( column_names, @@ -1696,7 +1714,7 @@ void ReadFromFile::initializePipeline(QueryPipelineBuilder & pipeline, const Bui auto pipe = Pipe::unitePipes(std::move(pipes)); size_t output_ports = pipe.numOutputPorts(); - const bool parallelize_output = ctx->getSettingsRef().parallelize_output_from_storages; + const bool parallelize_output = ctx->getSettingsRef()[Setting::parallelize_output_from_storages]; if (parallelize_output && storage->parallelizeOutputAfterReading(ctx) && output_ports > 0 && output_ports < max_num_streams) pipe.resize(max_num_streams); @@ -1796,8 +1814,8 @@ public: write_buf = wrapWriteBufferWithCompressionMethod( std::move(naked_buffer), compression_method, - static_cast(settings.output_format_compression_level), - static_cast(settings.output_format_compression_zstd_window_log)); + static_cast(settings[Setting::output_format_compression_level]), + static_cast(settings[Setting::output_format_compression_zstd_window_log])); writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format_name, *write_buf, metadata_snapshot->getSampleBlock(), getContext(), format_settings); @@ -1955,7 +1973,7 @@ SinkToStoragePtr StorageFile::write( int flags = 0; - if (context->getSettingsRef().engine_file_truncate_on_insert) + if (context->getSettingsRef()[Setting::engine_file_truncate_on_insert]) flags |= O_TRUNC; bool has_wildcards = path_for_partitioned_write.find(PartitionedSink::PARTITION_ID_WILDCARD) != String::npos; @@ -1994,11 +2012,11 @@ SinkToStoragePtr StorageFile::write( fs::create_directories(fs::path(path).parent_path()); std::error_code error_code; - if (!context->getSettingsRef().engine_file_truncate_on_insert && !is_path_with_globs + if (!context->getSettingsRef()[Setting::engine_file_truncate_on_insert] && !is_path_with_globs && !FormatFactory::instance().checkIfFormatSupportAppend(format_name, context, format_settings) && fs::file_size(path, error_code) != 0 && !error_code) { - if (context->getSettingsRef().engine_file_allow_create_multiple_files) + if (context->getSettingsRef()[Setting::engine_file_allow_create_multiple_files]) { auto pos = path.find_first_of('.', path.find_last_of('/')); size_t index = paths.size(); @@ -2139,30 +2157,16 @@ void registerStorageFile(StorageFactory & factory) // session and user are ignored. if (factory_args.storage_def->settings) { - FormatFactorySettings user_format_settings; - - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = factory_args.getContext()->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - { - user_format_settings.set(change.name, change.value); - } - } + Settings settings = factory_args.getContext()->getSettingsCopy(); // Apply changes from SETTINGS clause, with validation. - user_format_settings.applyChanges( - factory_args.storage_def->settings->changes); + settings.applyChanges(factory_args.storage_def->settings->changes); - storage_args.format_settings = getFormatSettings( - factory_args.getContext(), user_format_settings); + storage_args.format_settings = getFormatSettings(factory_args.getContext(), settings); } else { - storage_args.format_settings = getFormatSettings( - factory_args.getContext()); + storage_args.format_settings = getFormatSettings(factory_args.getContext()); } if (engine_args_ast.size() == 1) /// Table in database @@ -2196,7 +2200,7 @@ void registerStorageFile(StorageFactory & factory) literal->value.safeGet(), source_path, storage_args.path_to_archive, - factory_args.getLocalContext()->getSettingsRef().allow_archive_path_syntax); + factory_args.getLocalContext()->getSettingsRef()[Setting::allow_archive_path_syntax]); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be path or file descriptor"); } diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index f3eb88f3207..23ee7a18b53 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -38,6 +38,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 preferred_block_size_bytes; +} namespace ErrorCodes { @@ -688,7 +692,7 @@ Pipe StorageGenerateRandom::read( } /// Correction of block size for wide tables. - size_t preferred_block_size_bytes = context->getSettingsRef().preferred_block_size_bytes; + size_t preferred_block_size_bytes = context->getSettingsRef()[Setting::preferred_block_size_bytes]; if (preferred_block_size_bytes) { size_t estimated_row_size_bytes = estimateValueSize(std::make_shared(block_header.getDataTypes()), max_array_length, max_string_length); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index efa15c382dd..9f8a8cc1a3c 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -32,6 +32,16 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool any_join_distinct_right_table_keys; + extern const SettingsBool join_any_take_last_row; + extern const SettingsOverflowMode join_overflow_mode; + extern const SettingsBool join_use_nulls; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_rows_in_join; + extern const SettingsUInt64 max_bytes_in_join; +} namespace ErrorCodes { @@ -82,7 +92,7 @@ RWLockImpl::LockHolder StorageJoin::tryLockTimedWithContext(const RWLock & lock, { const String query_id = context ? context->getInitialQueryId() : RWLockImpl::NO_QUERY; const std::chrono::milliseconds acquire_timeout - = context ? context->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); + = context ? context->getSettingsRef()[Setting::lock_acquire_timeout] : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); return tryLockTimed(lock, type, query_id, acquire_timeout); } @@ -90,7 +100,7 @@ RWLockImpl::LockHolder StorageJoin::tryLockForCurrentQueryTimedWithContext(const { const String query_id = context ? context->getInitialQueryId() : RWLockImpl::NO_QUERY; const std::chrono::milliseconds acquire_timeout - = context ? context->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); + = context ? context->getSettingsRef()[Setting::lock_acquire_timeout] : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); return lock->getLock(type, query_id, acquire_timeout, false); } @@ -316,13 +326,13 @@ size_t StorageJoin::getSize(ContextPtr context) const std::optional StorageJoin::totalRows(const Settings &settings) const { - TableLockHolder holder = tryLockTimed(rwlock, RWLockImpl::Read, RWLockImpl::NO_QUERY, settings.lock_acquire_timeout); + TableLockHolder holder = tryLockTimed(rwlock, RWLockImpl::Read, RWLockImpl::NO_QUERY, settings[Setting::lock_acquire_timeout]); return join->getTotalRowCount(); } std::optional StorageJoin::totalBytes(const Settings &settings) const { - TableLockHolder holder = tryLockTimed(rwlock, RWLockImpl::Read, RWLockImpl::NO_QUERY, settings.lock_acquire_timeout); + TableLockHolder holder = tryLockTimed(rwlock, RWLockImpl::Read, RWLockImpl::NO_QUERY, settings[Setting::lock_acquire_timeout]); return join->getTotalByteCount(); } @@ -357,12 +367,12 @@ void registerStorageJoin(StorageFactory & factory) const auto & settings = args.getContext()->getSettingsRef(); - auto join_use_nulls = settings.join_use_nulls; - auto max_rows_in_join = settings.max_rows_in_join; - auto max_bytes_in_join = settings.max_bytes_in_join; - auto join_overflow_mode = settings.join_overflow_mode; - auto join_any_take_last_row = settings.join_any_take_last_row; - auto old_any_join = settings.any_join_distinct_right_table_keys; + auto join_use_nulls = settings[Setting::join_use_nulls]; + auto max_rows_in_join = settings[Setting::max_rows_in_join]; + auto max_bytes_in_join = settings[Setting::max_bytes_in_join]; + auto join_overflow_mode = settings[Setting::join_overflow_mode]; + auto join_any_take_last_row = settings[Setting::join_any_take_last_row]; + auto old_any_join = settings[Setting::any_join_distinct_right_table_keys]; bool persistent = true; String disk_name = "default"; diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index a6be9f8da04..ecb300844d8 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -61,10 +61,21 @@ #include -#include +#include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 insert_keeper_max_retries; + extern const SettingsUInt64 insert_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 insert_keeper_retry_max_backoff_ms; + extern const SettingsBool keeper_map_strict_mode; + extern const SettingsUInt64 keeper_max_retries; + extern const SettingsUInt64 keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 keeper_retry_max_backoff_ms; + extern const SettingsUInt64 max_compress_block_size; +} namespace FailPoints { @@ -165,10 +176,7 @@ public: } } - void onFinish() override - { - finalize(/*strict*/ context->getSettingsRef().keeper_map_strict_mode); - } + void onFinish() override { finalize(/*strict*/ context->getSettingsRef()[Setting::keeper_map_strict_mode]); } template void finalize(bool strict) @@ -179,9 +187,9 @@ public: getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings.insert_keeper_max_retries, - settings.insert_keeper_retry_initial_backoff_ms, - settings.insert_keeper_retry_max_backoff_ms}, + settings[Setting::insert_keeper_max_retries], + settings[Setting::insert_keeper_retry_initial_backoff_ms], + settings[Setting::insert_keeper_retry_max_backoff_ms]}, context->getProcessListElement()}; zk_retry.retryLoop([&]() @@ -416,7 +424,8 @@ StorageKeeperMap::StorageKeeperMap( ZooKeeperRetriesControl zk_retry{ getName(), getLogger(getName()), - ZooKeeperRetriesInfo{settings.keeper_max_retries, settings.keeper_retry_initial_backoff_ms, settings.keeper_retry_max_backoff_ms}, + ZooKeeperRetriesInfo{ + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, context_->getProcessListElement()}; zk_retry.retryLoop( @@ -661,9 +670,7 @@ Pipe StorageKeeperMap::read( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings.keeper_max_retries, - settings.keeper_retry_initial_backoff_ms, - settings.keeper_retry_max_backoff_ms}, + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, context_->getProcessListElement()}; std::vector children; @@ -692,9 +699,7 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings.keeper_max_retries, - settings.keeper_retry_initial_backoff_ms, - settings.keeper_retry_max_backoff_ms}, + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, local_context->getProcessListElement()}; zk_retry.retryLoop([&] @@ -842,7 +847,8 @@ private: auto data_file_path = temp_dir / fs::path{file_path}.filename(); auto data_out_compressed = temp_disk->writeFile(data_file_path); - auto data_out = std::make_unique(*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size); + auto data_out = std::make_unique( + *data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size); std::vector data_children; { auto holder = with_retries->createRetriesControlHolder("getKeeperMapDataKeys"); @@ -929,7 +935,7 @@ void StorageKeeperMap::backupData(BackupEntriesCollector & backup_entries_collec } auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); - auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; + auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef()[Setting::max_compress_block_size]; auto with_retries = std::make_shared ( @@ -1132,9 +1138,7 @@ StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus(const ContextPtr getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings.keeper_max_retries, - settings.keeper_retry_initial_backoff_ms, - settings.keeper_retry_max_backoff_ms}, + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, local_context->getProcessListElement()}; zk_retry.retryLoop([&] @@ -1246,9 +1250,7 @@ Chunk StorageKeeperMap::getBySerializedKeys( getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings.keeper_max_retries, - settings.keeper_retry_initial_backoff_ms, - settings.keeper_retry_max_backoff_ms}, + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, local_context->getProcessListElement()}; zkutil::ZooKeeper::MultiTryGetResponse values; @@ -1333,7 +1335,7 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca if (commands.empty()) return; - bool strict = local_context->getSettingsRef().keeper_map_strict_mode; + bool strict = local_context->getSettingsRef()[Setting::keeper_map_strict_mode]; chassert(commands.size() == 1); @@ -1394,9 +1396,7 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca getName(), getLogger(getName()), ZooKeeperRetriesInfo{ - settings.keeper_max_retries, - settings.keeper_retry_initial_backoff_ms, - settings.keeper_retry_max_backoff_ms}, + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}, local_context->getProcessListElement()}; Coordination::Error status; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 303532dfeca..552e3ab4bc0 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -49,6 +49,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_compress_block_size; + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -585,7 +591,7 @@ StorageLog::StorageLog( , use_marks_file(engine_name == "Log") , marks_file_path(table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME) , file_checker(disk, table_path + "sizes.json") - , max_compress_block_size(context_->getSettingsRef().max_compress_block_size) + , max_compress_block_size(context_->getSettingsRef()[Setting::max_compress_block_size]) { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); @@ -788,9 +794,9 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & static std::chrono::seconds getLockTimeout(ContextPtr context) { const Settings & settings = context->getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); + Int64 lock_timeout = settings[Setting::lock_acquire_timeout].totalSeconds(); + if (settings[Setting::max_execution_time].totalSeconds() != 0 && settings[Setting::max_execution_time].totalSeconds() < lock_timeout) + lock_timeout = settings[Setting::max_execution_time].totalSeconds(); return std::chrono::seconds{lock_timeout}; } diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index e1256032493..2dee120ba13 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -40,6 +40,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsSeconds lock_acquire_timeout; +} namespace ErrorCodes { @@ -238,7 +243,7 @@ void StorageMaterializedView::read( { auto context = getInMemoryMetadataPtr()->getSQLSecurityOverriddenContext(local_context); auto storage = getTargetTable(); - auto lock = storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout); + auto lock = storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); auto target_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto target_storage_snapshot = storage->getStorageSnapshot(target_metadata_snapshot, context); @@ -296,7 +301,7 @@ SinkToStoragePtr StorageMaterializedView::write(const ASTPtr & query, const Stor { auto context = getInMemoryMetadataPtr()->getSQLSecurityOverriddenContext(local_context); auto storage = getTargetTable(); - auto lock = storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout); + auto lock = storage->lockForShare(context->getCurrentQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = storage->getInMemoryMetadataPtr(); auto storage_id = storage->getStorageID(); @@ -425,7 +430,7 @@ std::shared_ptr StorageMaterializedView::prepareRefresh(bool app insert_query->table_id = target_table; Block header; - if (refresh_context->getSettingsRef().allow_experimental_analyzer) + if (refresh_context->getSettingsRef()[Setting::allow_experimental_analyzer]) header = InterpreterSelectQueryAnalyzer::getSampleBlock(insert_query->select, refresh_context); else header = InterpreterSelectWithUnionQuery(insert_query->select, refresh_context, SelectQueryOptions()).getSampleBlock(); diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index be7ec5afae7..84eed6dbbe6 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -39,6 +39,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_compress_block_size; +} namespace ErrorCodes { @@ -484,7 +488,7 @@ void StorageMemory::backupData(BackupEntriesCollector & backup_entries_collector { auto temp_disk = backup_entries_collector.getContext()->getGlobalTemporaryVolume()->getDisk(0); const auto & read_settings = backup_entries_collector.getReadSettings(); - auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef().max_compress_block_size; + auto max_compress_block_size = backup_entries_collector.getContext()->getSettingsRef()[Setting::max_compress_block_size]; backup_entries_collector.addBackupEntries(std::make_shared( backup_entries_collector.getContext(), diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index f40aa8ae4e8..c32d9a2c213 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -67,6 +67,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsFloat max_streams_multiplier_for_merge_tables; +} namespace { @@ -369,7 +375,7 @@ void StorageMerge::read( /// What will be result structure depending on query processed stage in source tables? Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage); - if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage == QueryProcessingStage::Complete) + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer] && processed_stage == QueryProcessingStage::Complete) { /// Remove constants. /// For StorageDistributed some functions like `hostName` that are constants only for local queries. @@ -472,8 +478,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu { size_t tables_count = selected_tables.size(); Float64 num_streams_multiplier = std::min( - tables_count, - std::max(1UL, static_cast(context->getSettingsRef().max_streams_multiplier_for_merge_tables))); + tables_count, std::max(1UL, static_cast(context->getSettingsRef()[Setting::max_streams_multiplier_for_merge_tables]))); size_t num_streams = static_cast(requested_num_streams * num_streams_multiplier); // It's possible to have many tables read from merge, resize(num_streams) might open too many files at the same time. @@ -519,7 +524,7 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ size_t tables_count = selected_tables.size(); Float64 num_streams_multiplier - = std::min(tables_count, std::max(1UL, static_cast(context->getSettingsRef().max_streams_multiplier_for_merge_tables))); + = std::min(tables_count, std::max(1UL, static_cast(context->getSettingsRef()[Setting::max_streams_multiplier_for_merge_tables]))); size_t num_streams = static_cast(requested_num_streams * num_streams_multiplier); size_t remaining_streams = num_streams; @@ -590,7 +595,7 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ auto modified_query_info = getModifiedQueryInfo(modified_context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases); - if (!context->getSettingsRef().allow_experimental_analyzer) + if (!context->getSettingsRef()[Setting::allow_experimental_analyzer]) { auto storage_columns = storage_metadata_snapshot->getColumns(); auto syntax_result = TreeRewriter(context).analyzeSelect( @@ -1047,13 +1052,12 @@ void ReadFromMerge::addVirtualColumns( const StorageWithLockAndName & storage_with_lock) const { const auto & [database_name, _, storage, table_name] = storage_with_lock; - bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer; /// Add virtual columns if we don't already have them. Block plan_header = child.plan.getCurrentDataStream().header; - if (allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { String table_alias = modified_query_info.query_tree->as()->getJoinTree()->as()->getAlias(); @@ -1133,8 +1137,8 @@ QueryPipelineBuilderPtr ReadFromMerge::buildPipeline( if (!builder->initialized()) return builder; - bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer; - if (processed_stage > child.stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns)) + if (processed_stage > child.stage + || (context->getSettingsRef()[Setting::allow_experimental_analyzer] && processed_stage != QueryProcessingStage::FetchColumns)) { /** Materialization is needed, since from distributed storage the constants come materialized. * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, @@ -1168,7 +1172,7 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable( modified_select.setFinal(); } - bool allow_experimental_analyzer = modified_context->getSettingsRef().allow_experimental_analyzer; + bool use_analyzer = modified_context->getSettingsRef()[Setting::allow_experimental_analyzer]; auto storage_stage = storage->getQueryProcessingStage(modified_context, processed_stage, @@ -1201,13 +1205,13 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable( row_policy_data_opt->addStorageFilter(source_step_with_filter); } } - else if (processed_stage > storage_stage || allow_experimental_analyzer) + else if (processed_stage > storage_stage || use_analyzer) { /// Maximum permissible parallelism is streams_num modified_context->setSetting("max_threads", streams_num); modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - if (allow_experimental_analyzer) + if (use_analyzer) { /// Converting query to AST because types might be different in the source table. /// Need to resolve types again. @@ -1329,7 +1333,7 @@ StorageMerge::StorageListWithLocks ReadFromMerge::getSelectedTables( if (storage.get() != storage_merge.get()) { - auto table_lock = storage->lockForShare(query_context->getCurrentQueryId(), settings.lock_acquire_timeout); + auto table_lock = storage->lockForShare(query_context->getCurrentQueryId(), settings[Setting::lock_acquire_timeout]); res.emplace_back(iterator->databaseName(), storage, std::move(table_lock), iterator->name()); if (filter_by_table_virtual_column) table_name_virtual_column->insert(iterator->name()); @@ -1479,7 +1483,7 @@ void ReadFromMerge::convertAndFilterSourceStream( auto storage_sample_block = snapshot->metadata->getSampleBlock(); auto pipe_columns = before_block_header.getNamesAndTypesList(); - if (local_context->getSettingsRef().allow_experimental_analyzer) + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { for (const auto & alias : aliases) { @@ -1522,7 +1526,7 @@ void ReadFromMerge::convertAndFilterSourceStream( ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name; - if (local_context->getSettingsRef().allow_experimental_analyzer + if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer] && (child.stage != QueryProcessingStage::FetchColumns || dynamic_cast(&snapshot->storage) != nullptr)) convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index ae3c0e869ad..049ca038401 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -42,7 +42,22 @@ namespace DB { - +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_suspicious_primary_key; + extern const SettingsUInt64 alter_sync; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsBool materialize_ttl_after_modify; + extern const SettingsUInt64 max_expanded_ast_elements; + extern const SettingsUInt64 max_partitions_per_insert_block; + extern const SettingsUInt64 max_table_size_to_drop; + extern const SettingsUInt64 mutations_sync; + extern const SettingsBool optimize_skip_merged_partitions; + extern const SettingsBool optimize_throw_if_noop; + extern const SettingsBool parallel_replicas_for_non_replicated_merge_tree; + extern const SettingsBool throw_on_unsupported_query_inside_transaction; +} namespace ErrorCodes { @@ -207,16 +222,16 @@ void StorageMergeTree::read( { const auto & settings = local_context->getSettingsRef(); /// reading step for parallel replicas with new analyzer is built in Planner, so don't do it here - if (local_context->canUseParallelReplicasOnInitiator() && settings.parallel_replicas_for_non_replicated_merge_tree - && !settings.allow_experimental_analyzer) + if (local_context->canUseParallelReplicasOnInitiator() && settings[Setting::parallel_replicas_for_non_replicated_merge_tree] + && !settings[Setting::allow_experimental_analyzer]) { ClusterProxy::executeQueryWithParallelReplicas( query_plan, getStorageID(), processed_stage, query_info.query, local_context, query_info.storage_limits); return; } - if (local_context->canUseParallelReplicasCustomKey() && settings.parallel_replicas_for_non_replicated_merge_tree - && !settings.allow_experimental_analyzer && local_context->getClientInfo().distributed_depth == 0) + if (local_context->canUseParallelReplicasCustomKey() && settings[Setting::parallel_replicas_for_non_replicated_merge_tree] + && !settings[Setting::allow_experimental_analyzer] && local_context->getClientInfo().distributed_depth == 0) { if (auto cluster = local_context->getClusterForParallelReplicas(); local_context->canUseParallelReplicasCustomKeyForCluster(*cluster)) @@ -243,8 +258,9 @@ void StorageMergeTree::read( } const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower() - && local_context->getSettingsRef().parallel_replicas_for_non_replicated_merge_tree - && (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas); + && local_context->getSettingsRef()[Setting::parallel_replicas_for_non_replicated_merge_tree] + && (!local_context->getSettingsRef()[Setting::allow_experimental_analyzer] + || query_info.current_table_chosen_for_reading_with_parallel_replicas); if (auto plan = reader.read( column_names, @@ -289,8 +305,7 @@ StorageMergeTree::write(const ASTPtr & /*query*/, const StorageMetadataPtr & met assertNotReadonly(); const auto & settings = local_context->getSettingsRef(); - return std::make_shared( - *this, metadata_snapshot, settings.max_partitions_per_insert_block, local_context); + return std::make_shared(*this, metadata_snapshot, settings[Setting::max_partitions_per_insert_block], local_context); } void StorageMergeTree::checkTableCanBeDropped(ContextPtr query_context) const @@ -301,9 +316,10 @@ void StorageMergeTree::checkTableCanBeDropped(ContextPtr query_context) const auto table_id = getStorageID(); const auto & query_settings = query_context->getSettingsRef(); - if (query_settings.max_table_size_to_drop.changed) + if (query_settings[Setting::max_table_size_to_drop].changed) { - getContext()->checkTableCanBeDropped(table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes(), query_settings.max_table_size_to_drop); + getContext()->checkTableCanBeDropped( + table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes(), query_settings[Setting::max_table_size_to_drop]); return; } @@ -323,7 +339,7 @@ void StorageMergeTree::alter( { assertNotReadonly(); - if (local_context->getCurrentTransaction() && local_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) + if (local_context->getCurrentTransaction() && local_context->getSettingsRef()[Setting::throw_on_unsupported_query_inside_transaction]) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ALTER METADATA is not supported inside transactions"); auto table_id = getStorageID(); @@ -333,14 +349,14 @@ void StorageMergeTree::alter( StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - auto maybe_mutation_commands = commands.getMutationCommands(new_metadata, query_settings.materialize_ttl_after_modify, local_context); + auto maybe_mutation_commands = commands.getMutationCommands(new_metadata, query_settings[Setting::materialize_ttl_after_modify], local_context); if (!maybe_mutation_commands.empty()) delayMutationOrThrowIfNeeded(nullptr, local_context); Int64 mutation_version = -1; commands.apply(new_metadata, local_context); - if (!query_settings.allow_suspicious_primary_key) + if (!query_settings[Setting::allow_suspicious_primary_key]) MergeTreeData::verifySortingKey(new_metadata.sorting_key); /// This alter can be performed at new_metadata level only @@ -395,7 +411,7 @@ void StorageMergeTree::alter( resetObjectColumnsFromActiveParts(parts_lock); } - if (!maybe_mutation_commands.empty() && query_settings.alter_sync > 0) + if (!maybe_mutation_commands.empty() && query_settings[Setting::alter_sync] > 0) waitForMutation(mutation_version, false); } @@ -641,18 +657,18 @@ void StorageMergeTree::mutate(const MutationCommands & commands, ContextPtr quer { /// It's important to serialize order of mutations with alter queries because /// they can depend on each other. - if (auto alter_lock = tryLockForAlter(query_context->getSettingsRef().lock_acquire_timeout); alter_lock == std::nullopt) + if (auto alter_lock = tryLockForAlter(query_context->getSettingsRef()[Setting::lock_acquire_timeout]); alter_lock == std::nullopt) { throw Exception( ErrorCodes::TIMEOUT_EXCEEDED, "Cannot start mutation in {}ms because some metadata-changing ALTER (MODIFY|RENAME|ADD|DROP) is currently executing. " "You can change this timeout with `lock_acquire_timeout` setting", - query_context->getSettingsRef().lock_acquire_timeout.totalMilliseconds()); + query_context->getSettingsRef()[Setting::lock_acquire_timeout].totalMilliseconds()); } version = startMutation(commands, query_context); } - if (query_context->getSettingsRef().mutations_sync > 0 || query_context->getCurrentTransaction()) + if (query_context->getSettingsRef()[Setting::mutations_sync] > 0 || query_context->getCurrentTransaction()) waitForMutation(version, false); } @@ -1203,7 +1219,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate( return {}; } - size_t max_ast_elements = getContext()->getSettingsRef().max_expanded_ast_elements; + size_t max_ast_elements = getContext()->getSettingsRef()[Setting::max_expanded_ast_elements]; auto future_part = std::make_shared(); if (storage_settings.get()->assign_part_uuids) @@ -1593,22 +1609,22 @@ bool StorageMergeTree::optimize( for (const String & partition_id : partition_ids) { if (!merge( - true, - partition_id, - true, - deduplicate, - deduplicate_by_columns, - cleanup, - txn, - disable_reason, - local_context->getSettingsRef().optimize_skip_merged_partitions)) + true, + partition_id, + true, + deduplicate, + deduplicate_by_columns, + cleanup, + txn, + disable_reason, + local_context->getSettingsRef()[Setting::optimize_skip_merged_partitions])) { constexpr auto message = "Cannot OPTIMIZE table: {}"; if (disable_reason.text.empty()) disable_reason = PreformattedMessage::create("unknown reason"); LOG_INFO(log, message, disable_reason.text); - if (local_context->getSettingsRef().optimize_throw_if_noop) + if (local_context->getSettingsRef()[Setting::optimize_throw_if_noop]) throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason.text); return false; } @@ -1621,22 +1637,22 @@ bool StorageMergeTree::optimize( partition_id = getPartitionIDFromQuery(partition, local_context); if (!merge( - true, - partition_id, - final, - deduplicate, - deduplicate_by_columns, - cleanup, - txn, - disable_reason, - local_context->getSettingsRef().optimize_skip_merged_partitions)) + true, + partition_id, + final, + deduplicate, + deduplicate_by_columns, + cleanup, + txn, + disable_reason, + local_context->getSettingsRef()[Setting::optimize_skip_merged_partitions])) { constexpr auto message = "Cannot OPTIMIZE table: {}"; if (disable_reason.text.empty()) disable_reason = PreformattedMessage::create("unknown reason"); LOG_INFO(log, message, disable_reason.text); - if (local_context->getSettingsRef().optimize_throw_if_noop) + if (local_context->getSettingsRef()[Setting::optimize_throw_if_noop]) throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason.text); return false; } @@ -2080,8 +2096,8 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con { assertNotReadonly(); - auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); - auto lock2 = source_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); + auto lock2 = source_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto merges_blocker = stopMergesAndWait(); auto source_metadata_snapshot = source_table->getInMemoryMetadataPtr(); auto my_metadata_snapshot = getInMemoryMetadataPtr(); @@ -2225,8 +2241,8 @@ void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const // Use the same back-pressure (delay/throw) logic as for INSERTs to be consistent and avoid possibility of exceeding part limits using MOVE PARTITION queries dest_table_storage->delayInsertOrThrowIfNeeded(nullptr, local_context, true); - auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); - auto lock2 = dest_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto lock1 = lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); + auto lock2 = dest_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto merges_blocker = stopMergesAndWait(); auto dest_metadata_snapshot = dest_table->getInMemoryMetadataPtr(); diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index e0818fafae9..d964cd33728 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 1d1a0ffdeaf..cefdc40df22 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 glob_expansion_max_elements; + extern const SettingsMySQLDataTypesSupport mysql_datatypes_support_level; + extern const SettingsUInt64 mysql_max_rows_to_insert; +} namespace ErrorCodes { @@ -80,7 +87,7 @@ ColumnsDescription StorageMySQL::getTableStructureFromData( const ContextPtr & context_) { const auto & settings = context_->getSettingsRef(); - const auto tables_and_columns = fetchTablesColumnsList(pool_, database, {table}, settings, settings.mysql_datatypes_support_level); + const auto tables_and_columns = fetchTablesColumnsList(pool_, database, {table}, settings, settings[Setting::mysql_datatypes_support_level]); const auto columns = tables_and_columns.find(table); if (columns == tables_and_columns.end()) @@ -253,7 +260,7 @@ SinkToStoragePtr StorageMySQL::write(const ASTPtr & /*query*/, const StorageMeta remote_database_name, remote_table_name, pool->get(), - local_context->getSettingsRef().mysql_max_rows_to_insert); + local_context->getSettingsRef()[Setting::mysql_max_rows_to_insert]); } StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( @@ -280,7 +287,7 @@ StorageMySQL::Configuration StorageMySQL::processNamedCollectionResult( } else { - size_t max_addresses = context_->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = context_->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase( configuration.addresses_expr, max_addresses, 3306); } @@ -321,7 +328,7 @@ StorageMySQL::Configuration StorageMySQL::getConfiguration(ASTs engine_args, Con engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context_); configuration.addresses_expr = checkAndGetLiteralArgument(engine_args[0], "host:port"); - size_t max_addresses = context_->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = context_->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase(configuration.addresses_expr, max_addresses, 3306); configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index e0a4af68824..0784154853e 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -49,9 +50,21 @@ #include +#include + namespace DB { +namespace Setting +{ + extern const SettingsBool external_table_functions_use_nulls; + extern const SettingsUInt64 glob_expansion_max_elements; + extern const SettingsUInt64 postgresql_connection_attempt_timeout; + extern const SettingsBool postgresql_connection_pool_auto_close_connection; + extern const SettingsUInt64 postgresql_connection_pool_retries; + extern const SettingsUInt64 postgresql_connection_pool_size; + extern const SettingsUInt64 postgresql_connection_pool_wait_timeout; +} namespace ErrorCodes { @@ -98,7 +111,7 @@ ColumnsDescription StoragePostgreSQL::getTableStructureFromData( const String & schema, const ContextPtr & context_) { - const bool use_nulls = context_->getSettingsRef().external_table_functions_use_nulls; + const bool use_nulls = context_->getSettingsRef()[Setting::external_table_functions_use_nulls]; auto connection_holder = pool_->get(); auto columns_info = fetchPostgreSQLTableStructure( connection_holder->get(), table, schema, use_nulls).physical_columns; @@ -540,7 +553,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::processNamedCollectionResult } else { - size_t max_addresses = context_->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = context_->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase( configuration.addresses_expr, max_addresses, 5432); } @@ -579,7 +592,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); configuration.addresses_expr = checkAndGetLiteralArgument(engine_args[0], "host:port"); - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = context->getSettingsRef()[Setting::glob_expansion_max_elements]; configuration.addresses = parseRemoteDescriptionForExternalDatabase(configuration.addresses_expr, max_addresses, 5432); if (configuration.addresses.size() == 1) @@ -610,12 +623,13 @@ void registerStoragePostgreSQL(StorageFactory & factory) { auto configuration = StoragePostgreSQL::getConfiguration(args.engine_args, args.getLocalContext()); const auto & settings = args.getContext()->getSettingsRef(); - auto pool = std::make_shared(configuration, - settings.postgresql_connection_pool_size, - settings.postgresql_connection_pool_wait_timeout, - settings.postgresql_connection_pool_retries, - settings.postgresql_connection_pool_auto_close_connection, - settings.postgresql_connection_attempt_timeout); + auto pool = std::make_shared( + configuration, + settings[Setting::postgresql_connection_pool_size], + settings[Setting::postgresql_connection_pool_wait_timeout], + settings[Setting::postgresql_connection_pool_retries], + settings[Setting::postgresql_connection_pool_auto_close_connection], + settings[Setting::postgresql_connection_attempt_timeout]); return std::make_shared( args.table_id, diff --git a/src/Storages/StorageRedis.cpp b/src/Storages/StorageRedis.cpp index 1a275320f43..80bf54e70a8 100644 --- a/src/Storages/StorageRedis.cpp +++ b/src/Storages/StorageRedis.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 865a0cbe506..1d1267d4261 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -140,6 +140,34 @@ namespace CurrentMetrics namespace DB { +namespace Setting +{ + extern const SettingsBool async_insert_deduplicate; + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_suspicious_primary_key; + extern const SettingsUInt64 alter_sync; + extern const SettingsBool async_query_sending_for_remote; + extern const SettingsBool async_socket_for_remote; + extern const SettingsBool insert_deduplicate; + extern const SettingsUInt64Auto insert_quorum; + extern const SettingsBool insert_quorum_parallel; + extern const SettingsMilliseconds insert_quorum_timeout; + extern const SettingsUInt64 keeper_max_retries; + extern const SettingsUInt64 keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 keeper_retry_max_backoff_ms; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_distributed_depth; + extern const SettingsUInt64 max_fetch_partition_retries_count; + extern const SettingsUInt64 max_partitions_per_insert_block; + extern const SettingsUInt64 max_table_size_to_drop; + extern const SettingsBool materialize_ttl_after_modify; + extern const SettingsUInt64 mutations_sync; + extern const SettingsBool optimize_skip_merged_partitions; + extern const SettingsBool optimize_throw_if_noop; + extern const SettingsSeconds receive_timeout; + extern const SettingsInt64 replication_wait_for_inactive_replica_timeout; + extern const SettingsUInt64 select_sequential_consistency; +} namespace FailPoints { @@ -5476,19 +5504,19 @@ void StorageReplicatedMergeTree::read( /// 1. To throw an exception if on a replica there are not all parts which have been written down on quorum of remaining replicas. /// 2. Do not read parts that have not yet been written to the quorum of the replicas. /// For this you have to synchronously go to ZooKeeper. - if (settings.select_sequential_consistency) + if (settings[Setting::select_sequential_consistency]) { readLocalSequentialConsistencyImpl(query_plan, column_names, storage_snapshot, query_info, local_context, max_block_size, num_streams); return; } /// reading step for parallel replicas with new analyzer is built in Planner, so don't do it here - if (local_context->canUseParallelReplicasOnInitiator() && !settings.allow_experimental_analyzer) + if (local_context->canUseParallelReplicasOnInitiator() && !settings[Setting::allow_experimental_analyzer]) { readParallelReplicasImpl(query_plan, column_names, query_info, local_context, processed_stage); return; } - if (local_context->canUseParallelReplicasCustomKey() && !settings.allow_experimental_analyzer + if (local_context->canUseParallelReplicasCustomKey() && !settings[Setting::allow_experimental_analyzer] && local_context->getClientInfo().distributed_depth == 0) { if (auto cluster = local_context->getClusterForParallelReplicas(); @@ -5557,7 +5585,7 @@ void StorageReplicatedMergeTree::readLocalImpl( const size_t num_streams) { const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower() - && (!local_context->getSettingsRef().allow_experimental_analyzer + && (!local_context->getSettingsRef()[Setting::allow_experimental_analyzer] || query_info.current_table_chosen_for_reading_with_parallel_replicas); auto plan = reader.read( @@ -5602,28 +5630,28 @@ void StorageReplicatedMergeTree::foreachActiveParts(Func && func, bool select_se std::optional StorageReplicatedMergeTree::totalRows(const Settings & settings) const { UInt64 res = 0; - foreachActiveParts([&res](auto & part) { res += part->rows_count; }, settings.select_sequential_consistency); + foreachActiveParts([&res](auto & part) { res += part->rows_count; }, settings[Setting::select_sequential_consistency]); return res; } std::optional StorageReplicatedMergeTree::totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr local_context) const { DataPartsVector parts; - foreachActiveParts([&](auto & part) { parts.push_back(part); }, local_context->getSettingsRef().select_sequential_consistency); + foreachActiveParts([&](auto & part) { parts.push_back(part); }, local_context->getSettingsRef()[Setting::select_sequential_consistency]); return totalRowsByPartitionPredicateImpl(filter_actions_dag, local_context, parts); } std::optional StorageReplicatedMergeTree::totalBytes(const Settings & settings) const { UInt64 res = 0; - foreachActiveParts([&res](auto & part) { res += part->getBytesOnDisk(); }, settings.select_sequential_consistency); + foreachActiveParts([&res](auto & part) { res += part->getBytesOnDisk(); }, settings[Setting::select_sequential_consistency]); return res; } std::optional StorageReplicatedMergeTree::totalBytesUncompressed(const Settings & settings) const { UInt64 res = 0; - foreachActiveParts([&res](auto & part) { res += part->getBytesUncompressedOnDisk(); }, settings.select_sequential_consistency); + foreachActiveParts([&res](auto & part) { res += part->getBytesUncompressedOnDisk(); }, settings[Setting::select_sequential_consistency]); return res; } @@ -5654,26 +5682,31 @@ SinkToStoragePtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, con const auto storage_settings_ptr = getSettings(); const Settings & query_settings = local_context->getSettingsRef(); - bool deduplicate = storage_settings_ptr->replicated_deduplication_window != 0 && query_settings.insert_deduplicate; - bool async_deduplicate = async_insert && query_settings.async_insert_deduplicate && storage_settings_ptr->replicated_deduplication_window_for_async_inserts != 0 && query_settings.insert_deduplicate; + bool deduplicate = storage_settings_ptr->replicated_deduplication_window != 0 && query_settings[Setting::insert_deduplicate]; + bool async_deduplicate = async_insert && query_settings[Setting::async_insert_deduplicate] + && storage_settings_ptr->replicated_deduplication_window_for_async_inserts != 0 && query_settings[Setting::insert_deduplicate]; if (async_deduplicate) return std::make_shared( - *this, metadata_snapshot, query_settings.insert_quorum.valueOr(0), - query_settings.insert_quorum_timeout.totalMilliseconds(), - query_settings.max_partitions_per_insert_block, - query_settings.insert_quorum_parallel, + *this, + metadata_snapshot, + query_settings[Setting::insert_quorum].valueOr(0), + query_settings[Setting::insert_quorum_timeout].totalMilliseconds(), + query_settings[Setting::max_partitions_per_insert_block], + query_settings[Setting::insert_quorum_parallel], deduplicate, - query_settings.insert_quorum.is_auto, + query_settings[Setting::insert_quorum].is_auto, local_context); // TODO: should we also somehow pass list of columns to deduplicate on to the ReplicatedMergeTreeSink? return std::make_shared( - *this, metadata_snapshot, query_settings.insert_quorum.valueOr(0), - query_settings.insert_quorum_timeout.totalMilliseconds(), - query_settings.max_partitions_per_insert_block, - query_settings.insert_quorum_parallel, + *this, + metadata_snapshot, + query_settings[Setting::insert_quorum].valueOr(0), + query_settings[Setting::insert_quorum_timeout].totalMilliseconds(), + query_settings[Setting::max_partitions_per_insert_block], + query_settings[Setting::insert_quorum_parallel], deduplicate, - query_settings.insert_quorum.is_auto, + query_settings[Setting::insert_quorum].is_auto, local_context); } @@ -5724,7 +5757,8 @@ std::optional StorageReplicatedMergeTree::distributedWriteFromClu QueryProcessingStage::Complete, extension); - QueryPipeline remote_pipeline(std::make_shared(remote_query_executor, false, settings.async_socket_for_remote, settings.async_query_sending_for_remote)); + QueryPipeline remote_pipeline(std::make_shared( + remote_query_executor, false, settings[Setting::async_socket_for_remote], settings[Setting::async_query_sending_for_remote])); remote_pipeline.complete(std::make_shared(remote_query_executor->getHeader())); pipeline.addCompletedPipeline(std::move(remote_pipeline)); @@ -5741,7 +5775,7 @@ std::optional StorageReplicatedMergeTree::distributedWrite(const return {}; const Settings & settings = local_context->getSettingsRef(); - if (settings.max_distributed_depth && local_context->getClientInfo().distributed_depth >= settings.max_distributed_depth) + if (settings[Setting::max_distributed_depth] && local_context->getClientInfo().distributed_depth >= settings[Setting::max_distributed_depth]) throw Exception(ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH, "Maximum distributed depth exceeded"); auto & select = query.select->as(); @@ -5790,7 +5824,7 @@ bool StorageReplicatedMergeTree::optimize( { /// NOTE: exclusive lock cannot be used here, since this may lead to deadlock (see comments below), /// but it should be safe to use non-exclusive to avoid dropping parts that may be required for processing queue. - auto table_lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto table_lock = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); assertNotReadonly(); @@ -5816,7 +5850,7 @@ bool StorageReplicatedMergeTree::optimize( { PreformattedMessage message = fmt_string.format(std::forward(args)...); LOG_DEBUG(log, message); - if (query_context->getSettingsRef().optimize_throw_if_noop) + if (query_context->getSettingsRef()[Setting::optimize_throw_if_noop]) throw Exception(std::move(message), ErrorCodes::CANNOT_ASSIGN_OPTIMIZE); return false; }; @@ -5866,8 +5900,14 @@ bool StorageReplicatedMergeTree::optimize( else { select_decision = merger_mutator.selectAllPartsToMergeWithinPartition( - future_merged_part, can_merge, partition_id, final, metadata_snapshot, NO_TRANSACTION_PTR, - disable_reason, query_context->getSettingsRef().optimize_skip_merged_partitions); + future_merged_part, + can_merge, + partition_id, + final, + metadata_snapshot, + NO_TRANSACTION_PTR, + disable_reason, + query_context->getSettingsRef()[Setting::optimize_skip_merged_partitions]); } /// If there is nothing to merge then we treat this merge as successful (needed for optimize final optimization) @@ -5881,7 +5921,7 @@ bool StorageReplicatedMergeTree::optimize( /// Here we trying to have a similar behaviour to ordinary MergeTree: if some merges are already in progress - let's wait for them to finish. /// This way `optimize final` won't just silently be a noop (if also `optimize_throw_if_noop=false`), but will wait for the active merges and repeat an attempt to schedule final merge. /// This guarantees are enough for tests, because there we have full control over insertions. - const auto wait_timeout = query_context->getSettingsRef().receive_timeout.totalMilliseconds() / max_retries; + const auto wait_timeout = query_context->getSettingsRef()[Setting::receive_timeout].totalMilliseconds() / max_retries; /// DEFAULT (and not LIGHTWEIGHT) because merges are not condidered lightweight; empty `source_replicas` means "all replicas" waitForProcessingQueue(wait_timeout, SyncReplicaMode::DEFAULT, {}); continue; @@ -6106,7 +6146,7 @@ void StorageReplicatedMergeTree::alter( return; } - if (!query_settings.allow_suspicious_primary_key) + if (!query_settings[Setting::allow_suspicious_primary_key]) { StorageInMemoryMetadata future_metadata = getInMemoryMetadata(); commands.apply(future_metadata, query_context); @@ -6243,10 +6283,8 @@ void StorageReplicatedMergeTree::alter( alter_entry->alter_version = new_metadata_version; alter_entry->create_time = time(nullptr); - auto maybe_mutation_commands = commands.getMutationCommands( - *current_metadata, - query_settings.materialize_ttl_after_modify, - query_context); + auto maybe_mutation_commands + = commands.getMutationCommands(*current_metadata, query_settings[Setting::materialize_ttl_after_modify], query_context); bool have_mutation = !maybe_mutation_commands.empty(); alter_entry->have_mutation = have_mutation; @@ -6368,7 +6406,7 @@ void StorageReplicatedMergeTree::alter( { LOG_DEBUG(log, "Metadata changes applied. Will wait for data changes."); merge_selecting_task->schedule(); - waitMutation(*mutation_znode, query_settings.alter_sync); + waitMutation(*mutation_znode, query_settings[Setting::alter_sync]); LOG_DEBUG(log, "Data changes applied."); } } @@ -6612,9 +6650,18 @@ PartitionCommandsResultInfo StorageReplicatedMergeTree::attachPartition( MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); /// TODO Allow to use quorum here. - ReplicatedMergeTreeSink output(*this, metadata_snapshot, /* quorum */ 0, /* quorum_timeout_ms */ 0, /* max_parts_per_block */ 0, - /* quorum_parallel */ false, query_context->getSettingsRef().insert_deduplicate, - /* majority_quorum */ false, query_context, /* is_attach */ true, /* allow_attach_while_readonly */ true); + ReplicatedMergeTreeSink output( + *this, + metadata_snapshot, + /* quorum */ 0, + /* quorum_timeout_ms */ 0, + /* max_parts_per_block */ 0, + /* quorum_parallel */ false, + query_context->getSettingsRef()[Setting::insert_deduplicate], + /* majority_quorum */ false, + query_context, + /* is_attach */ true, + /* allow_attach_while_readonly */ true); for (size_t i = 0; i < loaded_parts.size(); ++i) { @@ -6642,9 +6689,10 @@ void StorageReplicatedMergeTree::checkTableCanBeDropped(ContextPtr query_context auto table_id = getStorageID(); const auto & query_settings = query_context->getSettingsRef(); - if (query_settings.max_table_size_to_drop.changed) + if (query_settings[Setting::max_table_size_to_drop].changed) { - getContext()->checkTableCanBeDropped(table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes(), query_settings.max_table_size_to_drop); + getContext()->checkTableCanBeDropped( + table_id.database_name, table_id.table_name, getTotalActiveSizeInBytes(), query_settings[Setting::max_table_size_to_drop]); return; } @@ -6819,8 +6867,8 @@ void StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry( void StorageReplicatedMergeTree::waitForLogEntryToBeProcessedIfNecessary(const ReplicatedMergeTreeLogEntryData & entry, ContextPtr query_context, const String & error_context) { /// If necessary, wait until the operation is performed on itself or on all replicas. - Int64 wait_for_inactive_timeout = query_context->getSettingsRef().replication_wait_for_inactive_replica_timeout; - if (query_context->getSettingsRef().alter_sync == 1) + Int64 wait_for_inactive_timeout = query_context->getSettingsRef()[Setting::replication_wait_for_inactive_replica_timeout]; + if (query_context->getSettingsRef()[Setting::alter_sync] == 1) { bool finished = tryWaitForReplicaToProcessLogEntry(zookeeper_path, replica_name, entry, wait_for_inactive_timeout); if (!finished) @@ -6829,7 +6877,7 @@ void StorageReplicatedMergeTree::waitForLogEntryToBeProcessedIfNecessary(const R "most likely because the replica was shut down.", error_context, entry.znode_name); } } - else if (query_context->getSettingsRef().alter_sync == 2) + else if (query_context->getSettingsRef()[Setting::alter_sync] == 2) { waitForAllReplicasToProcessLogEntry(zookeeper_path, entry, wait_for_inactive_timeout, error_context); } @@ -7368,7 +7416,7 @@ void StorageReplicatedMergeTree::fetchPartition( if (try_no) LOG_INFO(log, "Some of parts ({}) are missing. Will try to fetch covering parts.", missing_parts.size()); - if (try_no >= query_context->getSettingsRef().max_fetch_partition_retries_count) + if (try_no >= query_context->getSettingsRef()[Setting::max_fetch_partition_retries_count]) throw Exception(ErrorCodes::TOO_MANY_RETRIES_TO_FETCH_PARTS, "Too many retries to fetch parts from {}:{}", from_zookeeper_name, best_replica_path); @@ -7575,7 +7623,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, Conte merge_selecting_task->schedule(); - waitMutation(mutation_entry.znode_name, query_context->getSettingsRef().mutations_sync); + waitMutation(mutation_entry.znode_name, query_context->getSettingsRef()[Setting::mutations_sync]); } void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const @@ -7784,7 +7832,8 @@ void StorageReplicatedMergeTree::forcefullyRemoveBrokenOutdatedPartFromZooKeeper bool exists = false; String part_path = replica_path + "/parts/" + part_name; const auto & settings = getContext()->getSettingsRef(); - ZooKeeperRetriesInfo retries_info{settings.keeper_max_retries, settings.keeper_retry_initial_backoff_ms, settings.keeper_retry_max_backoff_ms}; + ZooKeeperRetriesInfo retries_info{ + settings[Setting::keeper_max_retries], settings[Setting::keeper_retry_initial_backoff_ms], settings[Setting::keeper_retry_max_backoff_ms]}; ZooKeeperRetriesControl retries_ctl("outdatedPartExists", log.load(), retries_info, nullptr); retries_ctl.retryLoop([&]() { exists = getZooKeeper()->exists(part_path); }); @@ -7961,7 +8010,7 @@ void StorageReplicatedMergeTree::clearLockedBlockNumbersInPartition( Stopwatch time_waiting; const auto & stop_waiting = [this, &time_waiting]() { - auto timeout = getContext()->getSettingsRef().lock_acquire_timeout.value.seconds(); + auto timeout = getContext()->getSettingsRef()[Setting::lock_acquire_timeout].value.seconds(); return partial_shutdown_called || (timeout < time_waiting.elapsedSeconds()); }; zookeeper.waitForDisappear(paths_to_get[i], stop_waiting); @@ -8031,8 +8080,8 @@ void StorageReplicatedMergeTree::replacePartitionFrom( const StoragePtr & source_table, const ASTPtr & partition, bool replace, ContextPtr query_context) { /// First argument is true, because we possibly will add new data to current table. - auto lock1 = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); - auto lock2 = source_table->lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto lock1 = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); + auto lock2 = source_table->lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); const auto storage_settings_ptr = getSettings(); const auto source_metadata_snapshot = source_table->getInMemoryMetadataPtr(); @@ -8367,8 +8416,8 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta // Use the same back-pressure (delay/throw) logic as for INSERTs to be consistent and avoid possibility of exceeding part limits using MOVE PARTITION queries dest_table_storage->delayInsertOrThrowIfNeeded(nullptr, query_context, true); - auto lock1 = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); - auto lock2 = dest_table->lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef().lock_acquire_timeout); + auto lock1 = lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); + auto lock2 = dest_table->lockForShare(query_context->getCurrentQueryId(), query_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto storage_settings_ptr = getSettings(); auto dest_metadata_snapshot = dest_table->getInMemoryMetadataPtr(); diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index c892cca4523..ddc39cfb110 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -48,6 +48,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 max_compress_block_size; + extern const SettingsSeconds max_execution_time; +} namespace ErrorCodes { @@ -175,8 +181,7 @@ class StripeLogSink final : public SinkToStorage public: using WriteLock = std::unique_lock; - explicit StripeLogSink( - StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_, WriteLock && lock_) + explicit StripeLogSink(StorageStripeLog & storage_, const StorageMetadataPtr & metadata_snapshot_, WriteLock && lock_) : SinkToStorage(metadata_snapshot_->getSampleBlock()) , storage(storage_) , metadata_snapshot(metadata_snapshot_) @@ -291,7 +296,7 @@ StorageStripeLog::StorageStripeLog( , data_file_path(table_path + "data.bin") , index_file_path(table_path + "index.mrk") , file_checker(disk, table_path + "sizes.json") - , max_compress_block_size(context_->getSettingsRef().max_compress_block_size) + , max_compress_block_size(context_->getSettingsRef()[Setting::max_compress_block_size]) , log(getLogger("StorageStripeLog")) { StorageInMemoryMetadata storage_metadata; @@ -353,9 +358,9 @@ void StorageStripeLog::rename(const String & new_path_to_table_data, const Stora static std::chrono::seconds getLockTimeout(ContextPtr local_context) { const Settings & settings = local_context->getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); + Int64 lock_timeout = settings[Setting::lock_acquire_timeout].totalSeconds(); + if (settings[Setting::max_execution_time].totalSeconds() != 0 && settings[Setting::max_execution_time].totalSeconds() < lock_timeout) + lock_timeout = settings[Setting::max_execution_time].totalSeconds(); return std::chrono::seconds{lock_timeout}; } diff --git a/src/Storages/StorageTimeSeries.cpp b/src/Storages/StorageTimeSeries.cpp index 3ff57aaf3e5..a42758d6822 100644 --- a/src/Storages/StorageTimeSeries.cpp +++ b/src/Storages/StorageTimeSeries.cpp @@ -20,6 +20,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_time_series_table; +} namespace ErrorCodes { @@ -126,7 +130,7 @@ StorageTimeSeries::StorageTimeSeries( : IStorage(table_id) , WithContext(local_context->getGlobalContext()) { - if (mode <= LoadingStrictnessLevel::CREATE && !local_context->getSettingsRef().allow_experimental_time_series_table) + if (mode <= LoadingStrictnessLevel::CREATE && !local_context->getSettingsRef()[Setting::allow_experimental_time_series_table]) { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental TimeSeries table engine " diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index da0911ec20b..42648ad73e6 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include +#include #include #include #include @@ -57,6 +59,25 @@ namespace ProfileEvents namespace DB { +namespace Setting +{ + extern const SettingsBool enable_url_encoding; + extern const SettingsBool engine_url_skip_empty_files; + extern const SettingsUInt64 glob_expansion_max_elements; + extern const SettingsUInt64 max_http_get_redirects; + extern const SettingsMaxThreads max_parsing_threads; + extern const SettingsUInt64 max_read_buffer_size; + extern const SettingsBool optimize_count_from_files; + extern const SettingsBool schema_inference_cache_require_modification_time_for_url; + extern const SettingsSchemaInferenceMode schema_inference_mode; + extern const SettingsBool schema_inference_use_cache_for_url; + extern const SettingsBool parallelize_output_from_storages; + extern const SettingsUInt64 output_format_compression_level; + extern const SettingsUInt64 output_format_compression_zstd_window_log; + extern const SettingsBool use_cache_for_count_from_files; + extern const SettingsInt64 zstd_window_log_max; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -102,7 +123,7 @@ String getSampleURI(String uri, ContextPtr context) { if (urlWithGlobs(uri)) { - auto uris = parseRemoteDescription(uri, 0, uri.size(), ',', context->getSettingsRef().glob_expansion_max_elements); + auto uris = parseRemoteDescription(uri, 0, uri.size(), ',', context->getSettingsRef()[Setting::glob_expansion_max_elements]); if (!uris.empty()) return uris[0]; } @@ -346,7 +367,7 @@ StorageURLSource::StorageURLSource( /// If file is empty and engine_url_skip_empty_files=1, skip it and go to the next file. } - while (getContext()->getSettingsRef().engine_url_skip_empty_files && uri_and_buf.second->eof()); + while (getContext()->getSettingsRef()[Setting::engine_url_skip_empty_files] && uri_and_buf.second->eof()); curr_uri = uri_and_buf.first; auto last_mod_time = uri_and_buf.second->tryGetLastModificationTime(); @@ -358,7 +379,7 @@ StorageURLSource::StorageURLSource( QueryPipelineBuilder builder; std::optional num_rows_from_cache = std::nullopt; - if (need_only_count && getContext()->getSettingsRef().use_cache_for_count_from_files) + if (need_only_count && getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) num_rows_from_cache = tryGetNumRowsFromCache(curr_uri.toString(), last_mod_time); if (num_rows_from_cache) @@ -471,7 +492,7 @@ Chunk StorageURLSource::generate() return chunk; } - if (input_format && getContext()->getSettingsRef().use_cache_for_count_from_files) + if (input_format && getContext()->getSettingsRef()[Setting::use_cache_for_count_from_files]) addNumRowsToCache(curr_uri.toString(), total_rows_in_file); pipeline->reset(); @@ -505,7 +526,7 @@ std::pair> StorageURLSource: for (; option != end; ++option) { bool skip_url_not_found_error = glob_url && read_settings.http_skip_not_found_url_for_globs && option == std::prev(end); - auto request_uri = Poco::URI(*option, context_->getSettingsRef().enable_url_encoding); + auto request_uri = Poco::URI(*option, context_->getSettingsRef()[Setting::enable_url_encoding]); for (const auto & [param, value] : params) request_uri.addQueryParameter(param, value); @@ -525,15 +546,15 @@ std::pair> StorageURLSource: .withSettings(read_settings) .withTimeouts(timeouts) .withHostFilter(&context_->getRemoteHostFilter()) - .withBufSize(settings.max_read_buffer_size) - .withRedirects(settings.max_http_get_redirects) + .withBufSize(settings[Setting::max_read_buffer_size]) + .withRedirects(settings[Setting::max_http_get_redirects]) .withOutCallback(callback) .withSkipNotFound(skip_url_not_found_error) .withHeaders(headers) .withDelayInit(delay_initialization) .create(credentials); - if (context_->getSettingsRef().engine_url_skip_empty_files && res->eof() && option != std::prev(end)) + if (context_->getSettingsRef()[Setting::engine_url_skip_empty_files] && res->eof() && option != std::prev(end)) { last_skipped_empty_res = {request_uri, std::move(res)}; continue; @@ -577,7 +598,7 @@ std::optional StorageURLSource::tryGetNumRowsFromCache(const String & ur /// Some URLs could not have Last-Modified header, in this case we cannot be sure that /// data wasn't changed after adding it's schema to cache. Use schema from cache only if /// special setting for this case is enabled. - if (!last_mod_time && !getContext()->getSettingsRef().schema_inference_cache_require_modification_time_for_url) + if (!last_mod_time && !getContext()->getSettingsRef()[Setting::schema_inference_cache_require_modification_time_for_url]) return 0; return last_mod_time; }; @@ -611,8 +632,8 @@ StorageURLSink::StorageURLSink( write_buf = wrapWriteBufferWithCompressionMethod( std::move(write_buffer), compression_method, - static_cast(settings.output_format_compression_level), - static_cast(settings.output_format_compression_zstd_window_log)); + static_cast(settings[Setting::output_format_compression_level]), + static_cast(settings[Setting::output_format_compression_zstd_window_log])); writer = FormatFactory::instance().getOutputFormat(format, *write_buf, sample_block, context, format_settings); } @@ -755,7 +776,7 @@ namespace { url_options_to_check.reserve(urls_to_check_.size()); for (const auto & url : urls_to_check_) - url_options_to_check.push_back(getFailoverOptions(url, getContext()->getSettingsRef().glob_expansion_max_elements)); + url_options_to_check.push_back(getFailoverOptions(url, getContext()->getSettingsRef()[Setting::glob_expansion_max_elements])); } Data next() override @@ -783,7 +804,7 @@ namespace } /// For default mode check cached columns for all urls on first iteration. - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::DEFAULT) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::DEFAULT) { for (const auto & options : url_options_to_check) { @@ -817,7 +838,7 @@ namespace return {nullptr, std::nullopt, format}; } - if (getContext()->getSettingsRef().schema_inference_mode == SchemaInferenceMode::UNION) + if (getContext()->getSettingsRef()[Setting::schema_inference_mode] == SchemaInferenceMode::UNION) { if (auto cached_schema = tryGetColumnsFromCache(url_options_to_check[current_index])) { @@ -841,18 +862,18 @@ namespace false); ++current_index; - } while (getContext()->getSettingsRef().engine_url_skip_empty_files && uri_and_buf.second->eof()); + } while (getContext()->getSettingsRef()[Setting::engine_url_skip_empty_files] && uri_and_buf.second->eof()); current_url_option = uri_and_buf.first.toString(); return {wrapReadBufferWithCompressionMethod( std::move(uri_and_buf.second), compression_method, - static_cast(getContext()->getSettingsRef().zstd_window_log_max)), std::nullopt, format}; + static_cast(getContext()->getSettingsRef()[Setting::zstd_window_log_max])), std::nullopt, format}; } void setNumRowsToLastFile(size_t num_rows) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_url) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_url]) return; auto key = getKeyForSchemaCache(current_url_option, *format, format_settings, getContext()); @@ -861,8 +882,8 @@ namespace void setSchemaToLastFile(const ColumnsDescription & columns) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_url - || getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::UNION) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_url] + || getContext()->getSettingsRef()[Setting::schema_inference_mode] != SchemaInferenceMode::UNION) return; auto key = getKeyForSchemaCache(current_url_option, *format, format_settings, getContext()); @@ -871,8 +892,8 @@ namespace void setResultingSchema(const ColumnsDescription & columns) override { - if (!getContext()->getSettingsRef().schema_inference_use_cache_for_url - || getContext()->getSettingsRef().schema_inference_mode != SchemaInferenceMode::DEFAULT) + if (!getContext()->getSettingsRef()[Setting::schema_inference_use_cache_for_url] + || getContext()->getSettingsRef()[Setting::schema_inference_mode] != SchemaInferenceMode::DEFAULT) return; for (const auto & options : url_options_to_check) @@ -908,14 +929,15 @@ namespace false, false); - return wrapReadBufferWithCompressionMethod(std::move(uri_and_buf.second), compression_method, static_cast(getContext()->getSettingsRef().zstd_window_log_max)); + return wrapReadBufferWithCompressionMethod( + std::move(uri_and_buf.second), compression_method, static_cast(getContext()->getSettingsRef()[Setting::zstd_window_log_max])); } private: std::optional tryGetColumnsFromCache(const Strings & urls) { auto context = getContext(); - if (!context->getSettingsRef().schema_inference_use_cache_for_url) + if (!context->getSettingsRef()[Setting::schema_inference_use_cache_for_url]) return std::nullopt; auto & schema_cache = StorageURL::getSchemaCache(getContext()); @@ -927,7 +949,7 @@ namespace /// Some URLs could not have Last-Modified header, in this case we cannot be sure that /// data wasn't changed after adding it's schema to cache. Use schema from cache only if /// special setting for this case is enabled. - if (!last_mod_time && !getContext()->getSettingsRef().schema_inference_cache_require_modification_time_for_url) + if (!last_mod_time && !getContext()->getSettingsRef()[Setting::schema_inference_cache_require_modification_time_for_url]) return 0; return last_mod_time; }; @@ -983,7 +1005,7 @@ std::pair IStorageURLBase::getTableStructureAndForma std::vector urls_to_check; if (urlWithGlobs(uri)) - urls_to_check = parseRemoteDescription(uri, 0, uri.size(), ',', context->getSettingsRef().glob_expansion_max_elements, "url"); + urls_to_check = parseRemoteDescription(uri, 0, uri.size(), ',', context->getSettingsRef()[Setting::glob_expansion_max_elements], "url"); else urls_to_check = {uri}; @@ -1108,7 +1130,7 @@ void IStorageURLBase::read( auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context)); bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) - && local_context->getSettingsRef().optimize_count_from_files; + && local_context->getSettingsRef()[Setting::optimize_count_from_files]; auto read_post_data_callback = getReadPOSTDataCallback( read_from_format_info.columns_description.getNamesOfPhysical(), @@ -1156,7 +1178,7 @@ void ReadFromURL::createIterator(const ActionsDAG::Node * predicate) return; } - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + size_t max_addresses = context->getSettingsRef()[Setting::glob_expansion_max_elements]; is_url_with_globs = urlWithGlobs(storage->uri); if (storage->distributed_processing) @@ -1219,7 +1241,7 @@ void ReadFromURL::initializePipeline(QueryPipelineBuilder & pipeline, const Buil Pipes pipes; pipes.reserve(num_streams); - const size_t max_parsing_threads = num_streams >= settings.max_parsing_threads ? 1 : (settings.max_parsing_threads / num_streams); + const size_t max_parsing_threads = num_streams >= settings[Setting::max_parsing_threads] ? 1 : (settings[Setting::max_parsing_threads] / num_streams); for (size_t i = 0; i < num_streams; ++i) { @@ -1250,7 +1272,7 @@ void ReadFromURL::initializePipeline(QueryPipelineBuilder & pipeline, const Buil auto pipe = Pipe::unitePipes(std::move(pipes)); size_t output_ports = pipe.numOutputPorts(); - const bool parallelize_output = settings.parallelize_output_from_storages; + const bool parallelize_output = settings[Setting::parallelize_output_from_storages]; if (parallelize_output && storage->parallelizeOutputAfterReading(context) && output_ports > 0 && output_ports < max_num_streams) pipe.resize(max_num_streams); @@ -1278,7 +1300,7 @@ void StorageURLWithFailover::read( auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context)); bool need_only_count = (query_info.optimize_trivial_count || read_from_format_info.requested_columns.empty()) - && local_context->getSettingsRef().optimize_count_from_files; + && local_context->getSettingsRef()[Setting::optimize_count_from_files]; auto read_post_data_callback = getReadPOSTDataCallback( read_from_format_info.columns_description.getNamesOfPhysical(), @@ -1371,8 +1393,8 @@ std::optional IStorageURLBase::tryGetLastModificationTime( .withSettings(context->getReadSettings()) .withTimeouts(getHTTPTimeouts(context)) .withHostFilter(&context->getRemoteHostFilter()) - .withBufSize(settings.max_read_buffer_size) - .withRedirects(settings.max_http_get_redirects) + .withBufSize(settings[Setting::max_read_buffer_size]) + .withRedirects(settings[Setting::max_http_get_redirects]) .withHeaders(headers) .withProxy(proxy_config) .create(credentials); @@ -1448,23 +1470,12 @@ FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Argum FormatSettings format_settings; if (args.storage_def->settings) { - FormatFactorySettings user_format_settings; - - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = args.getContext()->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - { - user_format_settings.set(change.name, change.value); - } - } + Settings settings = args.getContext()->getSettingsCopy(); // Apply changes from SETTINGS clause, with validation. - user_format_settings.applyChanges(args.storage_def->settings->changes); + settings.applyChanges(args.storage_def->settings->changes); - format_settings = getFormatSettings(args.getContext(), user_format_settings); + format_settings = getFormatSettings(args.getContext(), settings); } else { diff --git a/src/Storages/StorageURLCluster.cpp b/src/Storages/StorageURLCluster.cpp index 140413d78b0..04fd5fb9675 100644 --- a/src/Storages/StorageURLCluster.cpp +++ b/src/Storages/StorageURLCluster.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -29,6 +30,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 glob_expansion_max_elements; +} namespace ErrorCodes { @@ -91,7 +96,8 @@ void StorageURLCluster::updateQueryToSendIfNeeded(ASTPtr & query, const StorageS RemoteQueryExecutor::Extension StorageURLCluster::getTaskIteratorExtension(const ActionsDAG::Node * predicate, const ContextPtr & context) const { - auto iterator = std::make_shared(uri, context->getSettingsRef().glob_expansion_max_elements, predicate, getVirtualsList(), context); + auto iterator = std::make_shared( + uri, context->getSettingsRef()[Setting::glob_expansion_max_elements], predicate, getVirtualsList(), context); auto callback = std::make_shared([iter = std::move(iterator)]() mutable -> String { return iter->next(); }); return RemoteQueryExecutor::Extension{.task_iterator = std::move(callback)}; } diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 878998ebf12..cb438e0efa6 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -33,6 +33,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool extremes; + extern const SettingsUInt64 max_result_rows; + extern const SettingsUInt64 max_result_bytes; +} namespace ErrorCodes { @@ -98,9 +105,9 @@ ContextPtr getViewContext(ContextPtr context, const StorageSnapshotPtr & storage { auto view_context = storage_snapshot->metadata->getSQLSecurityOverriddenContext(context); Settings view_settings = view_context->getSettingsCopy(); - view_settings.max_result_rows = 0; - view_settings.max_result_bytes = 0; - view_settings.extremes = false; + view_settings[Setting::max_result_rows] = 0; + view_settings[Setting::max_result_bytes] = 0; + view_settings[Setting::extremes] = false; view_context->setSettings(view_settings); return view_context; } @@ -164,7 +171,7 @@ void StorageView::read( auto options = SelectQueryOptions(QueryProcessingStage::Complete, 0, false, query_info.settings_limit_offset_done); - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { InterpreterSelectQueryAnalyzer interpreter(current_inner_query, getViewContext(context, storage_snapshot), options, column_names); interpreter.addStorageLimits(*query_info.storage_limits); diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index fa23a574ade..93f0604be28 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -19,6 +19,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds http_receive_timeout; + extern const SettingsBool odbc_bridge_use_connection_pooling; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -180,10 +186,11 @@ namespace for (size_t i = 0; i < 3; ++i) engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], args.getLocalContext()); - BridgeHelperPtr bridge_helper = std::make_shared>(args.getContext(), - args.getContext()->getSettingsRef().http_receive_timeout.value, + BridgeHelperPtr bridge_helper = std::make_shared>( + args.getContext(), + args.getContext()->getSettingsRef()[Setting::http_receive_timeout].value, checkAndGetLiteralArgument(engine_args[0], "connection_string"), - args.getContext()->getSettingsRef().odbc_bridge_use_connection_pooling.value); + args.getContext()->getSettingsRef()[Setting::odbc_bridge_use_connection_pooling].value); return std::make_shared( args.table_id, checkAndGetLiteralArgument(engine_args[1], "database_name"), diff --git a/src/Storages/System/MutableColumnsAndConstraints.h b/src/Storages/System/MutableColumnsAndConstraints.h new file mode 100644 index 00000000000..eb8dc933592 --- /dev/null +++ b/src/Storages/System/MutableColumnsAndConstraints.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace DB +{ + +struct MutableColumnsAndConstraints +{ + MutableColumns & res_columns; + const SettingsConstraints & constraints; + + MutableColumnsAndConstraints(MutableColumns & res_columns_, const SettingsConstraints & constraints_) + : res_columns(res_columns_), constraints(constraints_) + { + } +}; +} diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index bc13cb77d5e..710ee503086 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -25,7 +25,10 @@ namespace DB { - +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} StorageSystemColumns::StorageSystemColumns(const StorageID & table_id_) : IStorage(table_id_) @@ -83,11 +86,16 @@ public: Storages storages_, ContextPtr context) : ISource(header_) - , columns_mask(std::move(columns_mask_)), max_block_size(max_block_size_) - , databases(std::move(databases_)), tables(std::move(tables_)), storages(std::move(storages_)) + , columns_mask(std::move(columns_mask_)) + , max_block_size(max_block_size_) + , databases(std::move(databases_)) + , tables(std::move(tables_)) + , storages(std::move(storages_)) , client_info_interface(context->getClientInfo().interface) - , total_tables(tables->size()), access(context->getAccess()) - , query_id(context->getCurrentQueryId()), lock_acquire_timeout(context->getSettingsRef().lock_acquire_timeout) + , total_tables(tables->size()) + , access(context->getAccess()) + , query_id(context->getCurrentQueryId()) + , lock_acquire_timeout(context->getSettingsRef()[Setting::lock_acquire_timeout]) { need_to_check_access_for_tables = !access->isGranted(AccessType::SHOW_COLUMNS); } diff --git a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp index ae1a233ea81..3f0bc765bf0 100644 --- a/src/Storages/System/StorageSystemDDLWorkerQueue.cpp +++ b/src/Storages/System/StorageSystemDDLWorkerQueue.cpp @@ -20,6 +20,13 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool allow_settings_after_format_in_insert; + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} enum class Status : uint8_t { @@ -76,11 +83,9 @@ static String clusterNameFromDDLQuery(ContextPtr context, const DDLTask & task) const auto & settings = context->getSettingsRef(); String description = fmt::format("from {}", task.entry_path); - ParserQuery parser_query(end, settings.allow_settings_after_format_in_insert); - ASTPtr query = parseQuery(parser_query, begin, end, description, - settings.max_query_size, - settings.max_parser_depth, - settings.max_parser_backtracks); + ParserQuery parser_query(end, settings[Setting::allow_settings_after_format_in_insert]); + ASTPtr query = parseQuery( + parser_query, begin, end, description, settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); String cluster_name; if (const auto * query_on_cluster = dynamic_cast(query.get())) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 2955aebeb7e..2de815accb0 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -10,6 +10,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool system_events_show_zero_values; +} ColumnsDescription StorageSystemErrors::getColumnsDescription() { @@ -30,7 +34,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, ContextPtr cont { auto add_row = [&](std::string_view name, size_t code, const auto & error, bool remote) { - if (error.count || context->getSettingsRef().system_events_show_zero_values) + if (error.count || context->getSettingsRef()[Setting::system_events_show_zero_values]) { size_t col_num = 0; res_columns[col_num++]->insert(name); diff --git a/src/Storages/System/StorageSystemEvents.cpp b/src/Storages/System/StorageSystemEvents.cpp index f74c316e3fc..d04587c581d 100644 --- a/src/Storages/System/StorageSystemEvents.cpp +++ b/src/Storages/System/StorageSystemEvents.cpp @@ -7,6 +7,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool system_events_show_zero_values; +} ColumnsDescription StorageSystemEvents::getColumnsDescription() { @@ -30,7 +34,7 @@ void StorageSystemEvents::fillData(MutableColumns & res_columns, ContextPtr cont { UInt64 value = ProfileEvents::global_counters[i]; - if (0 != value || context->getSettingsRef().system_events_show_zero_values) + if (0 != value || context->getSettingsRef()[Setting::system_events_show_zero_values]) { res_columns[0]->insert(ProfileEvents::getName(ProfileEvents::Event(i))); res_columns[1]->insert(value); diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index c87bdb6d26a..8d48d65ae13 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -17,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -35,6 +35,15 @@ constexpr auto * storage_uuid_column_name = "storage_uuid"; namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; +} + +StoragesInfoStreamBase::StoragesInfoStreamBase(ContextPtr context) + : query_id(context->getCurrentQueryId()), lock_timeout(context->getSettingsRef()[Setting::lock_acquire_timeout]), next_row(0), rows(0) +{ +} bool StorageSystemPartsBase::hasStateColumn(const Names & column_names, const StorageSnapshotPtr & storage_snapshot) { @@ -362,4 +371,10 @@ StorageSystemPartsBase::StorageSystemPartsBase(const StorageID & table_id_, Colu setVirtuals(std::move(virtuals)); } +bool StoragesInfoStreamBase::tryLockTable(StoragesInfo & info) +{ + info.table_lock = info.storage->tryLockForShare(query_id, lock_timeout); + // nullptr means table was dropped while acquiring the lock + return info.table_lock != nullptr; +} } diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index 3be73aeda17..665c1cb890f 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -39,9 +38,7 @@ struct StoragesInfo class StoragesInfoStreamBase { public: - explicit StoragesInfoStreamBase(ContextPtr context) - : query_id(context->getCurrentQueryId()), settings(context->getSettingsRef()), next_row(0), rows(0) - {} + explicit StoragesInfoStreamBase(ContextPtr context); StoragesInfoStreamBase(const StoragesInfoStreamBase&) = default; virtual ~StoragesInfoStreamBase() = default; @@ -89,16 +86,10 @@ public: return {}; } protected: - virtual bool tryLockTable(StoragesInfo & info) - { - info.table_lock = info.storage->tryLockForShare(query_id, settings.lock_acquire_timeout); - // nullptr means table was dropped while acquiring the lock - return info.table_lock != nullptr; - } + virtual bool tryLockTable(StoragesInfo & info); String query_id; - Settings settings; - + std::chrono::milliseconds lock_timeout; ColumnPtr database_column; ColumnPtr table_column; diff --git a/src/Storages/System/StorageSystemRemoteDataPaths.cpp b/src/Storages/System/StorageSystemRemoteDataPaths.cpp index 00b4824cc3d..783efee4cc4 100644 --- a/src/Storages/System/StorageSystemRemoteDataPaths.cpp +++ b/src/Storages/System/StorageSystemRemoteDataPaths.cpp @@ -20,6 +20,10 @@ namespace fs = std::filesystem; namespace DB { +namespace Setting +{ + extern const SettingsBool traverse_shadow_remote_data_paths; +} namespace ErrorCodes { @@ -216,7 +220,7 @@ bool SystemRemoteDataPathsSource::nextDisk() /// cases when children of a directory get deleted while traversal is running. current.names.push_back({"store", nullptr}); current.names.push_back({"data", nullptr}); - if (context->getSettingsRef().traverse_shadow_remote_data_paths) + if (context->getSettingsRef()[Setting::traverse_shadow_remote_data_paths]) current.names.push_back({"shadow", skipPredicateForShadowDir}); /// Start and move to the first file diff --git a/src/Storages/System/StorageSystemSettings.cpp b/src/Storages/System/StorageSystemSettings.cpp index f1e9d542fec..9309f10378e 100644 --- a/src/Storages/System/StorageSystemSettings.cpp +++ b/src/Storages/System/StorageSystemSettings.cpp @@ -1,11 +1,12 @@ #include +#include #include -#include #include +#include #include #include -#include +#include namespace DB @@ -42,49 +43,8 @@ void StorageSystemSettings::fillData(MutableColumns & res_columns, ContextPtr co auto constraints_and_current_profiles = context->getSettingsConstraintsAndCurrentProfiles(); const auto & constraints = constraints_and_current_profiles->constraints; - const auto fill_data_for_setting = [&](std::string_view setting_name, const auto & setting) - { - res_columns[1]->insert(setting.getValueString()); - res_columns[2]->insert(setting.isValueChanged()); - res_columns[3]->insert(setting.getDescription()); - - Field min, max; - SettingConstraintWritability writability = SettingConstraintWritability::WRITABLE; - constraints.get(settings, setting_name, min, max, writability); - - /// These two columns can accept strings only. - if (!min.isNull()) - min = Settings::valueToStringUtil(setting_name, min); - if (!max.isNull()) - max = Settings::valueToStringUtil(setting_name, max); - - res_columns[4]->insert(min); - res_columns[5]->insert(max); - res_columns[6]->insert(writability == SettingConstraintWritability::CONST); - res_columns[7]->insert(setting.getTypeName()); - res_columns[8]->insert(setting.getDefaultValueString()); - res_columns[10]->insert(setting.isObsolete()); - }; - - const auto & settings_to_aliases = Settings::Traits::settingsToAliases(); - for (const auto & setting : settings.all()) - { - const auto & setting_name = setting.getName(); - res_columns[0]->insert(setting_name); - - fill_data_for_setting(setting_name, setting); - res_columns[9]->insert(""); - - if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end()) - { - for (const auto alias : it->second) - { - res_columns[0]->insert(alias); - fill_data_for_setting(alias, setting); - res_columns[9]->insert(setting_name); - } - } - } + MutableColumnsAndConstraints params(res_columns, constraints); + settings.dumpToSystemSettingsColumns(params); } } diff --git a/src/Storages/System/StorageSystemSettingsProfileElements.cpp b/src/Storages/System/StorageSystemSettingsProfileElements.cpp index 40a669351b1..9481bfd97b0 100644 --- a/src/Storages/System/StorageSystemSettingsProfileElements.cpp +++ b/src/Storages/System/StorageSystemSettingsProfileElements.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index 6cc525b9340..0151bb83dde 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -38,6 +38,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsMilliseconds storage_system_stack_trace_pipe_read_timeout_ms; +} namespace ErrorCodes { @@ -288,7 +292,7 @@ public: , predicate(filter_dag ? filter_dag->getOutputs().at(0) : nullptr) , max_block_size(max_block_size_) , pipe_read_timeout_ms( - static_cast(context->getSettingsRef().storage_system_stack_trace_pipe_read_timeout_ms.totalMilliseconds())) + static_cast(context->getSettingsRef()[Setting::storage_system_stack_trace_pipe_read_timeout_ms].totalMilliseconds())) , log(log_) , proc_it("/proc/self/task") /// It shouldn't be possible to do concurrent reads from this table. diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 943ce9c317a..135a853777f 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 select_sequential_consistency; + extern const SettingsBool show_table_uuid_in_table_create_query_if_not_nil; +} namespace { @@ -392,8 +399,7 @@ protected: static const size_t DATA_PATHS_INDEX = 5; if (columns_mask[DATA_PATHS_INDEX]) { - lock = table->tryLockForShare(context->getCurrentQueryId(), - context->getSettingsRef().lock_acquire_timeout); + lock = table->tryLockForShare(context->getCurrentQueryId(), context->getSettingsRef()[Setting::lock_acquire_timeout]); if (!lock) // Table was dropped while acquiring the lock, skipping table continue; @@ -481,7 +487,7 @@ protected: ASTPtr ast = database->tryGetCreateTableQuery(table_name, context); auto * ast_create = ast ? ast->as() : nullptr; - if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil) + if (ast_create && !context->getSettingsRef()[Setting::show_table_uuid_in_table_create_query_if_not_nil]) { ast_create->uuid = UUIDHelpers::Nil; if (ast_create->targets) @@ -561,7 +567,7 @@ protected: } auto settings = context->getSettingsRef(); - settings.select_sequential_consistency = 0; + settings[Setting::select_sequential_consistency] = 0; if (columns_mask[src_index++]) { auto total_rows = table ? table->totalRows(settings) : std::nullopt; diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index 81d3b0fe659..8dab26c7eb3 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -37,6 +37,16 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_unrestricted_reads_from_keeper; + extern const SettingsFloat insert_keeper_fault_injection_probability; + extern const SettingsUInt64 insert_keeper_fault_injection_seed; + extern const SettingsUInt64 insert_keeper_max_retries; + extern const SettingsUInt64 insert_keeper_retry_initial_backoff_ms; + extern const SettingsUInt64 insert_keeper_retry_max_backoff_ms; + extern const SettingsMaxThreads max_download_threads; +} namespace ErrorCodes { @@ -477,7 +487,7 @@ void ReadFromSystemZooKeeper::applyFilters(ActionDAGNodes added_filter_nodes) { SourceStepWithFilter::applyFilters(added_filter_nodes); - paths = extractPath(added_filter_nodes.nodes, context, context->getSettingsRef().allow_unrestricted_reads_from_keeper); + paths = extractPath(added_filter_nodes.nodes, context, context->getSettingsRef()[Setting::allow_unrestricted_reads_from_keeper]); } @@ -506,9 +516,9 @@ Chunk SystemZooKeeperSource::generate() /// Use insert settings for now in order not to introduce new settings. /// Hopefully insert settings will also be unified and replaced with some generic retry settings. ZooKeeperRetriesInfo retries_seetings( - settings.insert_keeper_max_retries, - settings.insert_keeper_retry_initial_backoff_ms, - settings.insert_keeper_retry_max_backoff_ms); + settings[Setting::insert_keeper_max_retries], + settings[Setting::insert_keeper_retry_initial_backoff_ms], + settings[Setting::insert_keeper_retry_max_backoff_ms]); /// Handles reconnects when needed auto get_zookeeper = [&] () @@ -516,15 +526,16 @@ Chunk SystemZooKeeperSource::generate() if (!zookeeper || zookeeper->expired()) { zookeeper = ZooKeeperWithFaultInjection::createInstance( - settings.insert_keeper_fault_injection_probability, - settings.insert_keeper_fault_injection_seed, + settings[Setting::insert_keeper_fault_injection_probability], + settings[Setting::insert_keeper_fault_injection_seed], context->getZooKeeper(), - "", nullptr); + "", + nullptr); } return zookeeper; }; - const Int64 max_inflight_requests = std::max(1, context->getSettingsRef().max_download_threads.value); + const Int64 max_inflight_requests = std::max(1, context->getSettingsRef()[Setting::max_download_threads].value); struct ListTask { diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index d674f054632..8ab4a8ceb72 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -24,6 +24,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_codecs; + extern const SettingsBool allow_suspicious_codecs; + extern const SettingsBool allow_suspicious_ttl_expressions; + extern const SettingsBool enable_zstd_qat_codec; + extern const SettingsBool enable_deflate_qpl_codec; +} namespace ErrorCodes { @@ -339,11 +347,11 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec, context->getSettingsRef().enable_zstd_qat_codec); + ttl_element->recompression_codec, {}, !context->getSettingsRef()[Setting::allow_suspicious_codecs], context->getSettingsRef()[Setting::allow_experimental_codecs], context->getSettingsRef()[Setting::enable_deflate_qpl_codec], context->getSettingsRef()[Setting::enable_zstd_qat_codec]); } } - checkTTLExpression(expression, result.result_column, is_attach || context->getSettingsRef().allow_suspicious_ttl_expressions); + checkTTLExpression(expression, result.result_column, is_attach || context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]); return result; } @@ -435,7 +443,7 @@ TTLTableDescription TTLTableDescription::parse(const String & str, const Columns ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); FunctionNameNormalizer::visit(ast.get()); - return getTTLForTableFromAST(ast, columns, context, primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions); + return getTTLForTableFromAST(ast, columns, context, primary_key, context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]); } } diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp index 14bf8ac8c13..da4ea9ea279 100644 --- a/src/Storages/VirtualColumnUtils.cpp +++ b/src/Storages/VirtualColumnUtils.cpp @@ -57,6 +57,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool use_hive_partitioning; +} namespace ErrorCodes { @@ -171,7 +175,7 @@ VirtualColumnsDescription getVirtualsForFileLikeStorage(ColumnsDescription & sto const auto & type = pair.getTypeInStorage(); if (storage_columns.has(name)) { - if (!context->getSettingsRef().use_hive_partitioning) + if (!context->getSettingsRef()[Setting::use_hive_partitioning]) return; if (storage_columns.size() == 1) @@ -188,7 +192,7 @@ VirtualColumnsDescription getVirtualsForFileLikeStorage(ColumnsDescription & sto for (const auto & item : getCommonVirtualsForFileLikeStorage()) add_virtual(item); - if (context->getSettingsRef().use_hive_partitioning) + if (context->getSettingsRef()[Setting::use_hive_partitioning]) { auto map = parseHivePartitioningKeysAndValues(path); auto format_settings = format_settings_ ? *format_settings_ : getFormatSettings(context); @@ -283,7 +287,7 @@ void addRequestedFileLikeStorageVirtualsToChunk( VirtualsForFileLikeStorage virtual_values, ContextPtr context) { std::unordered_map hive_map; - if (context->getSettingsRef().use_hive_partitioning) + if (context->getSettingsRef()[Setting::use_hive_partitioning]) hive_map = parseHivePartitioningKeysAndValues(virtual_values.path); for (const auto & virtual_column : requested_virtual_columns) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 4a20a07ae89..467015145f3 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -62,6 +62,20 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; + extern const SettingsBool allow_experimental_window_view; + extern const SettingsBool insert_null_as_default; + extern const SettingsSeconds lock_acquire_timeout; + extern const SettingsUInt64 min_insert_block_size_bytes; + extern const SettingsUInt64 min_insert_block_size_rows; + extern const SettingsBool use_concurrency_control; + extern const SettingsSeconds wait_for_window_view_fire_signal_timeout; + extern const SettingsSeconds window_view_clean_interval; + extern const SettingsSeconds window_view_heartbeat_interval; +} + namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; @@ -634,13 +648,14 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) { return std::make_shared(current_header); }); - builder.addSimpleTransform([&](const Block & current_header) - { - return std::make_shared( - current_header, - getContext()->getSettingsRef().min_insert_block_size_rows, - getContext()->getSettingsRef().min_insert_block_size_bytes); - }); + builder.addSimpleTransform( + [&](const Block & current_header) + { + return std::make_shared( + current_header, + getContext()->getSettingsRef()[Setting::min_insert_block_size_rows], + getContext()->getSettingsRef()[Setting::min_insert_block_size_bytes]); + }); auto header = builder.getHeader(); auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); @@ -698,7 +713,7 @@ inline void StorageWindowView::fire(UInt32 watermark) /* allow_materialized */ false, /* no_squash */ false, /* no_destination */ false, - /* async_isnert */ false); + /* async_insert */ false); auto block_io = interpreter.execute(); auto pipe = Pipe(std::make_shared(blocks, header)); @@ -708,7 +723,7 @@ inline void StorageWindowView::fire(UInt32 watermark) block_io.pipeline.getHeader().getNamesAndTypesList(), getTargetTable()->getInMemoryMetadataPtr()->getColumns(), getContext(), - getContext()->getSettingsRef().insert_null_as_default); + getContext()->getSettingsRef()[Setting::insert_null_as_default]); auto adding_missing_defaults_actions = std::make_shared(std::move(adding_missing_defaults_dag)); pipe.addSimpleTransform([&](const Block & stream_header) { @@ -1144,7 +1159,7 @@ void StorageWindowView::read( return; auto storage = getTargetTable(); - auto lock = storage->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto lock = storage->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto target_metadata_snapshot = storage->getInMemoryMetadataPtr(); auto target_storage_snapshot = storage->getStorageSnapshot(target_metadata_snapshot, local_context); @@ -1197,7 +1212,7 @@ Pipe StorageWindowView::watch( window_view_timezone, has_limit, limit, - local_context->getSettingsRef().window_view_heartbeat_interval.totalSeconds()); + local_context->getSettingsRef()[Setting::window_view_heartbeat_interval].totalSeconds()); std::lock_guard lock(fire_signal_mutex); watch_streams.push_back(reader); @@ -1216,10 +1231,10 @@ StorageWindowView::StorageWindowView( : IStorage(table_id_) , WithContext(context_->getGlobalContext()) , log(getLogger(fmt::format("StorageWindowView({}.{})", table_id_.database_name, table_id_.table_name))) - , fire_signal_timeout_s(context_->getSettingsRef().wait_for_window_view_fire_signal_timeout.totalSeconds()) - , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds()) + , fire_signal_timeout_s(context_->getSettingsRef()[Setting::wait_for_window_view_fire_signal_timeout].totalSeconds()) + , clean_interval_usec(context_->getSettingsRef()[Setting::window_view_clean_interval].totalMicroseconds()) { - if (context_->getSettingsRef().allow_experimental_analyzer) + if (context_->getSettingsRef()[Setting::allow_experimental_analyzer]) disabled_due_to_analyzer = true; if (mode <= LoadingStrictnessLevel::CREATE) @@ -1594,13 +1609,14 @@ void StorageWindowView::writeIntoWindowView( }); #endif - builder.addSimpleTransform([&](const Block & current_header) - { - return std::make_shared( - current_header, - local_context->getSettingsRef().min_insert_block_size_rows, - local_context->getSettingsRef().min_insert_block_size_bytes); - }); + builder.addSimpleTransform( + [&](const Block & current_header) + { + return std::make_shared( + current_header, + local_context->getSettingsRef()[Setting::min_insert_block_size_rows], + local_context->getSettingsRef()[Setting::min_insert_block_size_bytes]); + }); if (!window_view.is_proctime) { @@ -1641,8 +1657,7 @@ void StorageWindowView::writeIntoWindowView( #endif auto inner_table = window_view.getInnerTable(); - auto lock = inner_table->lockForShare( - local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout); + auto lock = inner_table->lockForShare(local_context->getCurrentQueryId(), local_context->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = inner_table->getInMemoryMetadataPtr(); auto output = inner_table->write(window_view.getMergeableQuery(), metadata_snapshot, local_context, /*async_insert=*/false); output->addTableLock(lock); @@ -1673,7 +1688,7 @@ void StorageWindowView::writeIntoWindowView( }); auto executor = builder.execute(); - executor->execute(builder.getNumThreads(), local_context->getSettingsRef().use_concurrency_control); + executor->execute(builder.getNumThreads(), local_context->getSettingsRef()[Setting::use_concurrency_control]); LOG_TRACE(window_view.log, "Wrote {} rows into inner table ({})", block_rows, inner_table->getStorageID().getFullTableName()); } @@ -1780,22 +1795,26 @@ StoragePtr StorageWindowView::getTargetTable() const void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) const { - if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().allow_experimental_analyzer)) + if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef()[Setting::allow_experimental_analyzer])) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Experimental WINDOW VIEW feature is not supported " "in the current infrastructure for query analysis (the setting 'allow_experimental_analyzer')"); } void registerStorageWindowView(StorageFactory & factory) { - factory.registerStorage("WindowView", [](const StorageFactory::Arguments & args) - { - if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef().allow_experimental_window_view) - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Experimental WINDOW VIEW feature " - "is not enabled (the setting 'allow_experimental_window_view')"); + factory.registerStorage( + "WindowView", + [](const StorageFactory::Arguments & args) + { + if (args.mode <= LoadingStrictnessLevel::CREATE && !args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_window_view]) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Experimental WINDOW VIEW feature " + "is not enabled (the setting 'allow_experimental_window_view')"); - return std::make_shared(args.table_id, args.getLocalContext(), args.query, args.columns, args.comment, args.mode); - }); + return std::make_shared( + args.table_id, args.getLocalContext(), args.query, args.columns, args.comment, args.mode); + }); } } diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 69e8f0a7880..28e82abbec0 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -26,6 +26,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsDistributedProductMode distributed_product_mode; + extern const SettingsUInt64 min_external_table_block_size_rows; + extern const SettingsUInt64 min_external_table_block_size_bytes; + extern const SettingsBool parallel_replicas_prefer_local_join; + extern const SettingsBool prefer_global_in_and_join; +} namespace ErrorCodes { @@ -182,7 +190,7 @@ private: if (!distributed_valid_for_rewrite) return; - auto distributed_product_mode = getSettings().distributed_product_mode; + auto distributed_product_mode = getSettings()[Setting::distributed_product_mode]; if (distributed_product_mode == DistributedProductMode::LOCAL) { @@ -194,7 +202,7 @@ private: auto replacement_table_expression = std::make_shared(std::move(storage), getContext()); replacement_map.emplace(table_node.get(), std::move(replacement_table_expression)); } - else if ((distributed_product_mode == DistributedProductMode::GLOBAL || getSettings().prefer_global_in_and_join) && + else if ((distributed_product_mode == DistributedProductMode::GLOBAL || getSettings()[Setting::prefer_global_in_and_join]) && !in_function_or_join_stack.empty()) { auto * in_or_join_node_to_modify = in_function_or_join_stack.back().query_node.get(); @@ -289,8 +297,8 @@ TableNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node, auto build_pipeline_settings = BuildQueryPipelineSettings::fromContext(mutable_context); auto builder = query_plan.buildQueryPipeline(optimization_settings, build_pipeline_settings); - size_t min_block_size_rows = mutable_context->getSettingsRef().min_external_table_block_size_rows; - size_t min_block_size_bytes = mutable_context->getSettingsRef().min_external_table_block_size_bytes; + size_t min_block_size_rows = mutable_context->getSettingsRef()[Setting::min_external_table_block_size_rows]; + size_t min_block_size_bytes = mutable_context->getSettingsRef()[Setting::min_external_table_block_size_bytes]; auto squashing = std::make_shared(builder->getHeader(), min_block_size_rows, min_block_size_bytes); builder->resize(1); @@ -441,7 +449,7 @@ public: { if (auto * join_node = node->as()) { - bool prefer_local_join = getContext()->getSettingsRef().parallel_replicas_prefer_local_join; + bool prefer_local_join = getContext()->getSettingsRef()[Setting::parallel_replicas_prefer_local_join]; bool should_use_global_join = !prefer_local_join || !allStoragesAreMergeTree(join_node->getRightTableExpression()); if (should_use_global_join) join_node->setLocality(JoinLocality::Global); diff --git a/src/Storages/examples/CMakeLists.txt b/src/Storages/examples/CMakeLists.txt index 4f221efbd2b..0d6accbc895 100644 --- a/src/Storages/examples/CMakeLists.txt +++ b/src/Storages/examples/CMakeLists.txt @@ -1,8 +1,8 @@ clickhouse_add_executable (merge_selector merge_selector.cpp) -target_link_libraries (merge_selector PRIVATE dbms) +target_link_libraries (merge_selector PRIVATE dbms clickhouse_functions) clickhouse_add_executable (merge_selector2 merge_selector2.cpp) -target_link_libraries (merge_selector2 PRIVATE dbms) +target_link_libraries (merge_selector2 PRIVATE dbms clickhouse_functions) clickhouse_add_executable (get_current_inserts_in_replicated get_current_inserts_in_replicated.cpp) target_link_libraries (get_current_inserts_in_replicated PRIVATE dbms clickhouse_common_config clickhouse_common_zookeeper clickhouse_functions) diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index 1408e120bc5..fc9cc795e2c 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -19,6 +19,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_result_bytes; + extern const SettingsUInt64 max_result_rows; +} namespace ErrorCodes { @@ -66,8 +73,8 @@ ColumnsDescription getStructureOfRemoteTableInShard( /// since this is a service query and should not lead to query failure. { Settings new_settings = new_context->getSettingsCopy(); - new_settings.max_result_rows = 0; - new_settings.max_result_bytes = 0; + new_settings[Setting::max_result_rows] = 0; + new_settings[Setting::max_result_bytes] = 0; new_context->setSettings(new_settings); } @@ -114,7 +121,7 @@ ColumnsDescription getStructureOfRemoteTableInShard( String expr_str = (*default_expr)[i].safeGet(); column.default_desc.expression = parseQuery( expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression", - 0, settings.max_parser_depth, settings.max_parser_backtracks); + 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); } res.add(column); diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 251470a17a8..0cf4f706b9c 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -14,12 +14,17 @@ #include #include #include - #include +#include + namespace DB { +namespace Setting +{ + extern const SettingsBool external_table_strict_query; +} namespace ErrorCodes { @@ -292,7 +297,7 @@ String transformQueryForExternalDatabaseImpl( ContextPtr context, std::optional limit) { - bool strict = context->getSettingsRef().external_table_strict_query; + bool strict = context->getSettingsRef()[Setting::external_table_strict_query]; auto select = std::make_shared(); diff --git a/src/TableFunctions/Hive/TableFunctionHive.cpp b/src/TableFunctions/Hive/TableFunctionHive.cpp index 759807d7a4f..2124d5c3dcd 100644 --- a/src/TableFunctions/Hive/TableFunctionHive.cpp +++ b/src/TableFunctions/Hive/TableFunctionHive.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} namespace ErrorCodes { @@ -99,9 +106,9 @@ StoragePtr TableFunctionHive::executeImpl( partition_by_parser, "(" + partition_by_def + ")", "partition by declaration list", - settings.max_query_size, - settings.max_parser_depth, - settings.max_parser_backtracks); + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks]); StoragePtr storage; storage = std::make_shared( hive_metastore_url, diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index 83d0ac4ef41..91ff2a2f445 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -24,6 +24,13 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool external_table_functions_use_nulls; + extern const SettingsSeconds http_receive_timeout; + extern const SettingsBool odbc_bridge_use_connection_pooling; +} + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -135,7 +142,11 @@ void ITableFunctionXDBC::startBridgeIfNot(ContextPtr context) const { if (!helper) { - helper = createBridgeHelper(context, context->getSettingsRef().http_receive_timeout.value, connection_string, context->getSettingsRef().odbc_bridge_use_connection_pooling.value); + helper = createBridgeHelper( + context, + context->getSettingsRef()[Setting::http_receive_timeout].value, + connection_string, + context->getSettingsRef()[Setting::odbc_bridge_use_connection_pooling].value); helper->startBridgeSync(); } } @@ -151,7 +162,7 @@ ColumnsDescription ITableFunctionXDBC::getActualTableStructure(ContextPtr contex columns_info_uri.addQueryParameter("schema", schema_name); columns_info_uri.addQueryParameter("table", remote_table_name); - bool use_nulls = context->getSettingsRef().external_table_functions_use_nulls; + bool use_nulls = context->getSettingsRef()[Setting::external_table_functions_use_nulls]; columns_info_uri.addQueryParameter("external_table_functions_use_nulls", toString(use_nulls)); Poco::Net::HTTPBasicCredentials credentials{}; diff --git a/src/TableFunctions/TableFunctionExplain.cpp b/src/TableFunctions/TableFunctionExplain.cpp index 69d24c879bd..92764005fa6 100644 --- a/src/TableFunctions/TableFunctionExplain.cpp +++ b/src/TableFunctions/TableFunctionExplain.cpp @@ -18,6 +18,12 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_query_size; +} namespace ErrorCodes { @@ -99,8 +105,8 @@ void TableFunctionExplain::parseArguments(const ASTPtr & ast_function, ContextPt /// parse_only_internals_ = true - we don't want to parse `SET` keyword ParserSetQuery settings_parser(/* parse_only_internals_ = */ true); - ASTPtr settings_ast = parseQuery(settings_parser, settings_str, - settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + ASTPtr settings_ast = parseQuery( + settings_parser, settings_str, settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); explain_query->setSettings(std::move(settings_ast)); } diff --git a/src/TableFunctions/TableFunctionFactory.cpp b/src/TableFunctions/TableFunctionFactory.cpp index e505535ae76..f5381576a9a 100644 --- a/src/TableFunctions/TableFunctionFactory.cpp +++ b/src/TableFunctions/TableFunctionFactory.cpp @@ -11,6 +11,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool log_queries; +} namespace ErrorCodes { @@ -76,7 +80,7 @@ TableFunctionPtr TableFunctionFactory::tryGet( if (CurrentThread::isInitialized()) { auto query_context = CurrentThread::get().getQueryContext(); - if (query_context && query_context->getSettingsRef().log_queries) + if (query_context && query_context->getSettingsRef()[Setting::log_queries]) query_context->addQueryFactoriesInfo(Context::QueryLogFactories::TableFunction, name); } diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 5cd249f000d..9e7352033fc 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -15,6 +15,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_archive_path_syntax; + extern const SettingsString rename_files_after_processing; +} namespace ErrorCodes { @@ -26,7 +31,7 @@ void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, const ContextPtr if (context->getApplicationType() != Context::ApplicationType::LOCAL) { ITableFunctionFileLike::parseFirstArguments(arg, context); - StorageFile::parseFileSource(std::move(filename), filename, path_to_archive, context->getSettingsRef().allow_archive_path_syntax); + StorageFile::parseFileSource(std::move(filename), filename, path_to_archive, context->getSettingsRef()[Setting::allow_archive_path_syntax]); return; } @@ -43,7 +48,7 @@ void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, const ContextPtr fd = STDERR_FILENO; else StorageFile::parseFileSource( - std::move(filename), filename, path_to_archive, context->getSettingsRef().allow_archive_path_syntax); + std::move(filename), filename, path_to_archive, context->getSettingsRef()[Setting::allow_archive_path_syntax]); } else if (type == Field::Types::Int64 || type == Field::Types::UInt64) { @@ -83,7 +88,7 @@ StoragePtr TableFunctionFile::getStorage( columns, ConstraintsDescription{}, String{}, - global_context->getSettingsRef().rename_files_after_processing, + global_context->getSettingsRef()[Setting::rename_files_after_processing], path_to_archive, }; diff --git a/src/TableFunctions/TableFunctionFileCluster.cpp b/src/TableFunctions/TableFunctionFileCluster.cpp index 514fa3143d0..7093447beb1 100644 --- a/src/TableFunctions/TableFunctionFileCluster.cpp +++ b/src/TableFunctions/TableFunctionFileCluster.cpp @@ -9,6 +9,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsString rename_files_after_processing; +} StoragePtr TableFunctionFileCluster::getStorage( const String & /*source*/, const String & /*format_*/, const ColumnsDescription & columns, ContextPtr context, @@ -28,9 +32,8 @@ StoragePtr TableFunctionFileCluster::getStorage( columns, ConstraintsDescription{}, String{}, - context->getSettingsRef().rename_files_after_processing, - path_to_archive - }; + context->getSettingsRef()[Setting::rename_files_after_processing], + path_to_archive}; storage = std::make_shared(filename, context->getUserFilesPath(), true, args); } diff --git a/src/TableFunctions/TableFunctionFormat.cpp b/src/TableFunctions/TableFunctionFormat.cpp index 7e4fdea1ff3..fde9e07e8d4 100644 --- a/src/TableFunctions/TableFunctionFormat.cpp +++ b/src/TableFunctions/TableFunctionFormat.cpp @@ -25,6 +25,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_block_size; +} namespace ErrorCodes { @@ -101,7 +105,7 @@ Block TableFunctionFormat::parseData(const ColumnsDescription & columns, const S block.insert({name_and_type.type->createColumn(), name_and_type.type, name_and_type.name}); auto read_buf = std::make_unique(data); - auto input_format = context->getInputFormat(format_name, *read_buf, block, context->getSettingsRef().max_block_size); + auto input_format = context->getInputFormat(format_name, *read_buf, block, context->getSettingsRef()[Setting::max_block_size]); QueryPipelineBuilder builder; builder.init(Pipe(input_format)); if (columns.hasDefaults()) diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index ffd318f36f7..833a12c9b68 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -23,6 +23,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 external_storage_connect_timeout_sec; + extern const SettingsUInt64 external_storage_rw_timeout_sec; +} namespace ErrorCodes { @@ -67,8 +72,8 @@ void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, ContextPtr MySQLSettings mysql_settings; const auto & settings = context->getSettingsRef(); - mysql_settings.connect_timeout = settings.external_storage_connect_timeout_sec; - mysql_settings.read_write_timeout = settings.external_storage_rw_timeout_sec; + mysql_settings.connect_timeout = settings[Setting::external_storage_connect_timeout_sec]; + mysql_settings.read_write_timeout = settings[Setting::external_storage_rw_timeout_sec]; for (auto * it = args.begin(); it != args.end(); ++it) { diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index d690da9949a..81aec849f54 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -15,6 +15,14 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 postgresql_connection_attempt_timeout; + extern const SettingsBool postgresql_connection_pool_auto_close_connection; + extern const SettingsUInt64 postgresql_connection_pool_retries; + extern const SettingsUInt64 postgresql_connection_pool_size; + extern const SettingsUInt64 postgresql_connection_pool_wait_timeout; +} namespace ErrorCodes { @@ -79,11 +87,11 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, Contex const auto & settings = context->getSettingsRef(); connection_pool = std::make_shared( *configuration, - settings.postgresql_connection_pool_size, - settings.postgresql_connection_pool_wait_timeout, - settings.postgresql_connection_pool_retries, - settings.postgresql_connection_pool_auto_close_connection, - settings.postgresql_connection_attempt_timeout); + settings[Setting::postgresql_connection_pool_size], + settings[Setting::postgresql_connection_pool_wait_timeout], + settings[Setting::postgresql_connection_pool_retries], + settings[Setting::postgresql_connection_pool_auto_close_connection], + settings[Setting::postgresql_connection_attempt_timeout]); } } diff --git a/src/TableFunctions/TableFunctionRedis.cpp b/src/TableFunctions/TableFunctionRedis.cpp index aca751c2840..43a562df67d 100644 --- a/src/TableFunctions/TableFunctionRedis.cpp +++ b/src/TableFunctions/TableFunctionRedis.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 8a877ff0802..a9cdf1d3540 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsUInt64 table_function_remote_max_addresses; +} namespace ErrorCodes { @@ -247,7 +252,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr else { /// Create new cluster from the scratch - size_t max_addresses = context->getSettingsRef().table_function_remote_max_addresses; + size_t max_addresses = context->getSettingsRef()[Setting::table_function_remote_max_addresses]; std::vector shards = parseRemoteDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); std::vector> names; diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 57501df6d4d..a51fa045c47 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -12,6 +12,11 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -50,7 +55,7 @@ ColumnsDescription TableFunctionView::getActualTableStructure(ContextPtr context Block sample_block; - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context); else sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context); diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp index 935be6c1987..1699d5da302 100644 --- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp +++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp @@ -18,6 +18,10 @@ namespace DB { +namespace Setting +{ + extern const SettingsBool allow_experimental_analyzer; +} namespace ErrorCodes { @@ -114,7 +118,7 @@ bool TableFunctionViewIfPermitted::isPermitted(const ContextPtr & context, const try { - if (context->getSettingsRef().allow_experimental_analyzer) + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context); } diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 46593e85e45..c94f1d898c1 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -50,6 +50,29 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/n # Broken symlinks find -L $ROOT_PATH -type l 2>/dev/null | grep -v contrib && echo "^ Broken symlinks found" +# Duplicated or incorrect setting declarations +SETTINGS_FILE=$(mktemp) +cat $ROOT_PATH/src/Core/Settings.cpp $ROOT_PATH/src/Core/FormatFactorySettingsDeclaration.h | grep "M(" | awk '{print substr($2, 0, length($2) - 1) " " substr($1, 3, length($1) - 3) " SettingsDeclaration" }' > ${SETTINGS_FILE} +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep "extern const Settings" -T | awk '{print substr($5, 0, length($5) -1) " " substr($4, 9) " " substr($1, 0, length($1) - 1)}' >> ${SETTINGS_FILE} + +# Duplicate extern declarations for settings +awk '{if (seen[$0]++) print $3 " -> " $1 ;}' ${SETTINGS_FILE} | while read line; +do + echo "Found duplicated setting declaration in: $line" +done + +# Incorrect declarations for settings +for setting in $(awk '{print $1 " " $2}' ${SETTINGS_FILE} | sort | uniq | awk '{ print $1 }' | sort | uniq -d); +do + expected=$(grep "^$setting " ${SETTINGS_FILE} | grep SettingsDeclaration | awk '{ print $2 }') + grep "^$setting " ${SETTINGS_FILE} | grep -v " $expected" | awk '{ print $3 " found setting " $1 " with type " $2 }' | while read line; + do + echo "In $line but it should be $expected" + done +done + +rm ${SETTINGS_FILE} + # Unused/Undefined/Duplicates ErrorCodes/ProfileEvents/CurrentMetrics declare -A EXTERN_TYPES EXTERN_TYPES[ErrorCodes]=int @@ -469,5 +492,5 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | echo "If an exception has LOGICAL_ERROR code, there is no need to include the text 'Logical error' in the exception message, because then the phrase 'Logical error' will be printed twice." PATTERN="allow_"; -DIFF=$(comm -3 <(grep -o "\b$PATTERN\w*\b" $ROOT_PATH/src/Core/Settings.h | sort -u) <(grep -o -h "\b$PATTERN\w*\b" $ROOT_PATH/src/Databases/enableAllExperimentalSettings.cpp $ROOT_PATH/utils/check-style/experimental_settings_ignore.txt | sort -u)); +DIFF=$(comm -3 <(grep -o "\b$PATTERN\w*\b" $ROOT_PATH/src/Core/Settings.cpp | sort -u) <(grep -o -h "\b$PATTERN\w*\b" $ROOT_PATH/src/Databases/enableAllExperimentalSettings.cpp $ROOT_PATH/utils/check-style/experimental_settings_ignore.txt | sort -u)); [ -n "$DIFF" ] && echo "$DIFF" && echo "^^ Detected 'allow_*' settings that might need to be included in src/Databases/enableAllExperimentalSettings.cpp" && echo "Alternatively, consider adding an exception to utils/check-style/experimental_settings_ignore.txt" diff --git a/utils/checksum-for-compressed-block/CMakeLists.txt b/utils/checksum-for-compressed-block/CMakeLists.txt index da0434c0ffe..84378f6c1b7 100644 --- a/utils/checksum-for-compressed-block/CMakeLists.txt +++ b/utils/checksum-for-compressed-block/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable (checksum-for-compressed-block-find-bit-flips main.cpp) -target_link_libraries(checksum-for-compressed-block-find-bit-flips PRIVATE dbms) +target_link_libraries(checksum-for-compressed-block-find-bit-flips PRIVATE dbms clickhouse_functions) diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index f6c4cbd56b9..81f981f8126 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -1,2 +1,2 @@ clickhouse_add_executable (decompress_perf decompress_perf.cpp) -target_link_libraries(decompress_perf PRIVATE dbms ch_contrib::lz4) +target_link_libraries(decompress_perf PRIVATE dbms clickhouse_functions ch_contrib::lz4) diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index 0584e9e34c8..bbda26498bc 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,11 @@ namespace CurrentMetrics extern const Metric LocalThreadScheduled; } +namespace DB::Setting +{ + extern const SettingsUInt64 max_block_size; +} + namespace DB::ErrorCodes { extern const int CANNOT_BLOCK_SIGNAL; @@ -563,7 +569,7 @@ struct ZooKeeperRequestFromLogReader *file_read_buf, header_block, context, - context->getSettingsRef().max_block_size, + context->getSettingsRef()[DB::Setting::max_block_size], format_settings, 1, std::nullopt, diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt index c0b0b8a589d..842b63b6070 100644 --- a/utils/memcpy-bench/CMakeLists.txt +++ b/utils/memcpy-bench/CMakeLists.txt @@ -28,5 +28,5 @@ endif() set_source_files_properties(FastMemcpy.cpp PROPERTIES COMPILE_FLAGS "-Wno-old-style-cast") set_source_files_properties(FastMemcpy_Avx.cpp PROPERTIES COMPILE_FLAGS "-mavx -Wno-old-style-cast -Wno-cast-qual -Wno-cast-align") -target_link_libraries(memcpy-bench PRIVATE dbms boost::program_options) +target_link_libraries(memcpy-bench PRIVATE dbms clickhouse_functions boost::program_options)