From 0762e1a8901c9cdc798582cb7f3fa74eb02b1834 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 29 Mar 2021 02:23:20 +0400 Subject: [PATCH 001/202] Implement config parsing and actual support for user_dn_detection section and user_dn placeholder substitution Refactor some config parsing code Rename some arguments to better reflect their meaning Add documentation for user_dn_detection section and user_dn placeholder in config.xml and in docs --- .../external-authenticators/ldap.md | 42 +++++++- programs/server/config.xml | 47 +++++++- src/Access/ExternalAuthenticators.cpp | 101 +++++++++++++----- src/Access/ExternalAuthenticators.h | 6 +- src/Access/LDAPAccessStorage.cpp | 29 +---- src/Access/LDAPAccessStorage.h | 4 +- src/Access/LDAPClient.cpp | 92 ++++++++++++---- src/Access/LDAPClient.h | 17 ++- 8 files changed, 252 insertions(+), 86 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 1b65ecc968b..805d45e1b38 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -17,6 +17,7 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`. + localhost 636 @@ -31,6 +32,18 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`. /path/to/tls_ca_cert_dir ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + + + localhost + 389 + EXAMPLE\{user_name} + + CN=Users,DC=example,DC=com + (&(objectClass=user)(sAMAccountName={user_name})) + + no + ``` @@ -43,6 +56,15 @@ Note, that you can define multiple LDAP servers inside the `ldap_servers` sectio - `port` — LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise. - `bind_dn` — Template used to construct the DN to bind to. - The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt. +- `user_dn_detection` - Section with LDAP search parameters for detecting the actual user DN of the bound user. + - This is mainly used in search filters for further role mapping when the server is Active Directory. The resulting user DN will be used when replacing `{user_dn}` substrings wherever they are allowed. By default, user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected user DN value. + - `base_dn` - Template used to construct the base DN for the LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during the LDAP search. + - `scope` - Scope of the LDAP search. + - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). + - `search_filter` - Template used to construct the search filter for the LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. + - Note, that the special characters must be escaped properly in XML. - `verification_cooldown` — A period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. - `enable_tls` — A flag to trigger the use of the secure connection to the LDAP server. @@ -107,7 +129,7 @@ Goes into `config.xml`. - + my_ldap_server @@ -122,6 +144,18 @@ Goes into `config.xml`. clickhouse_ + + + + my_ad_server + + CN=Users,DC=example,DC=com + CN + subtree + (&(objectClass=group)(member={user_dn})) + clickhouse_ + + ``` @@ -137,13 +171,13 @@ Note that `my_ldap_server` referred in the `ldap` section inside the `user_direc - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged-in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - `base_dn` — Template used to construct the base DN for the LDAP search. - - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. - `scope` — Scope of the LDAP search. - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - `search_filter` — Template used to construct the search filter for the LDAP search. - - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}` and `{base_dn}` substrings of the template with the actual user name, bind DN and base DN during each LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, `{user_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, user DN, and base DN during each LDAP search. - Note, that the special characters must be escaped properly in XML. - - `attribute` — Attribute name whose values will be returned by the LDAP search. + - `attribute` — Attribute name whose values will be returned by the LDAP search. `cn`, by default. - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. The prefix will be removed from the original strings and the resulting strings will be treated as local role names. Empty by default. [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap/) diff --git a/programs/server/config.xml b/programs/server/config.xml index 4220ecbcacd..b6df1c42cc0 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -328,6 +328,20 @@ bind_dn - template used to construct the DN to bind to. The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual user name during each authentication attempt. + user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user. + This is mainly used in search filters for further role mapping when the server is Active Directory. The + resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default, + user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected + user DN value. + base_dn - template used to construct the base DN for the LDAP search. + The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings + of the template with the actual user name and bind DN during the LDAP search. + scope - scope of the LDAP search. + Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). + search_filter - template used to construct the search filter for the LDAP search. + The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' + substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. + Note, that the special characters must be escaped properly in XML. verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request. @@ -359,6 +373,17 @@ /path/to/tls_ca_cert_dir ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + Example (typical Active Directory with configured user DN detection for further role mapping): + + localhost + 389 + EXAMPLE\{user_name} + + CN=Users,DC=example,DC=com + (&(objectClass=user)(sAMAccountName={user_name})) + + no + --> @@ -410,15 +435,16 @@ There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be applied. base_dn - template used to construct the base DN for the LDAP search. - The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings - of the template with the actual user name and bind DN during each LDAP search. + The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' + substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. scope - scope of the LDAP search. Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). search_filter - template used to construct the search filter for the LDAP search. - The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' - substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. + The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and + '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during + each LDAP search. Note, that the special characters must be escaped properly in XML. - attribute - attribute name whose values will be returned by the LDAP search. + attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. @@ -437,6 +463,17 @@ clickhouse_ + Example (typical Active Directory with role mapping that relies on the detected user DN): + + my_ad_server + + CN=Users,DC=example,DC=com + CN + subtree + (&(objectClass=group)(member={user_dn})) + clickhouse_ + + --> diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 1cade973724..9eaf2a4b04b 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -20,13 +20,42 @@ namespace ErrorCodes namespace { -auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const String & name) +void parseLDAPSearchParams(LDAPClient::SearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) +{ + const bool has_base_dn = config.has(prefix + ".base_dn"); + const bool has_search_filter = config.has(prefix + ".search_filter"); + const bool has_attribute = config.has(prefix + ".attribute"); + const bool has_scope = config.has(prefix + ".scope"); + + if (has_base_dn) + params.base_dn = config.getString(prefix + ".base_dn"); + + if (has_search_filter) + params.search_filter = config.getString(prefix + ".search_filter"); + + if (has_attribute) + params.attribute = config.getString(prefix + ".attribute"); + + if (has_scope) + { + auto scope = config.getString(prefix + ".scope"); + boost::algorithm::to_lower(scope); + + if (scope == "base") params.scope = LDAPClient::SearchParams::Scope::BASE; + else if (scope == "one_level") params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL; + else if (scope == "subtree") params.scope = LDAPClient::SearchParams::Scope::SUBTREE; + else if (scope == "children") params.scope = LDAPClient::SearchParams::Scope::CHILDREN; + else + throw Exception("Invalid value for 'scope' field of LDAP search parameters in '" + prefix + + "' section, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS); + } +} + +void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConfiguration & config, const String & name) { if (name.empty()) throw Exception("LDAP server name cannot be empty", ErrorCodes::BAD_ARGUMENTS); - LDAPClient::Params params; - const String ldap_server_config = "ldap_servers." + name; const bool has_host = config.has(ldap_server_config + ".host"); @@ -34,6 +63,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str const bool has_bind_dn = config.has(ldap_server_config + ".bind_dn"); const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix"); const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix"); + const bool has_user_dn_detection = config.has(ldap_server_config + ".user_dn_detection"); const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown"); const bool has_enable_tls = config.has(ldap_server_config + ".enable_tls"); const bool has_tls_minimum_protocol_version = config.has(ldap_server_config + ".tls_minimum_protocol_version"); @@ -66,6 +96,14 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str params.bind_dn = auth_dn_prefix + "{user_name}" + auth_dn_suffix; } + if (has_user_dn_detection) + { + if (!params.user_dn_detection) + params.user_dn_detection = { .attribute = "dn" }; + + parseLDAPSearchParams(*params.user_dn_detection, config, ldap_server_config + ".user_dn_detection"); + } + if (has_verification_cooldown) params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")}; @@ -143,14 +181,10 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str } else params.port = (params.enable_tls == LDAPClient::Params::TLSEnable::YES ? 636 : 389); - - return params; } -auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config) +void parseKerberosParams(GSSAcceptorContext::Params & params, const Poco::Util::AbstractConfiguration & config) { - GSSAcceptorContext::Params params; - Poco::Util::AbstractConfiguration::Keys keys; config.keys("kerberos", keys); @@ -180,12 +214,20 @@ auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config) params.realm = config.getString("kerberos.realm", ""); params.principal = config.getString("kerberos.principal", ""); - - return params; } } +void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) +{ + parseLDAPSearchParams(params, config, prefix); + + const bool has_prefix = config.has(prefix + ".prefix"); + + if (has_prefix) + params.prefix = config.getString(prefix + ".prefix"); +} + void ExternalAuthenticators::reset() { std::scoped_lock lock(mutex); @@ -229,7 +271,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur { try { - ldap_client_params_blueprint.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name)); + ldap_client_params_blueprint.erase(ldap_server_name); + parseLDAPServer(ldap_client_params_blueprint.emplace(ldap_server_name, LDAPClient::Params{}).first->second, config, ldap_server_name); } catch (...) { @@ -240,7 +283,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur try { if (kerberos_keys_count > 0) - kerberos_params = parseKerberosParams(config); + parseKerberosParams(kerberos_params.emplace(), config); } catch (...) { @@ -249,7 +292,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur } bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const BasicCredentials & credentials, - const LDAPClient::SearchParamsList * search_params, LDAPClient::SearchResultsList * search_results) const + const LDAPClient::RoleSearchParamsList * role_search_params, LDAPClient::SearchResultsList * role_search_results) const { std::optional params; std::size_t params_hash = 0; @@ -267,9 +310,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B params->password = credentials.getPassword(); params->combineCoreHash(params_hash); - if (search_params) + if (role_search_params) { - for (const auto & params_instance : *search_params) + for (const auto & params_instance : *role_search_params) { params_instance.combineHash(params_hash); } @@ -301,14 +344,14 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B // Ensure that search_params are compatible. ( - search_params == nullptr ? - entry.last_successful_search_results.empty() : - search_params->size() == entry.last_successful_search_results.size() + role_search_params == nullptr ? + entry.last_successful_role_search_results.empty() : + role_search_params->size() == entry.last_successful_role_search_results.size() ) ) { - if (search_results) - *search_results = entry.last_successful_search_results; + if (role_search_results) + *role_search_results = entry.last_successful_role_search_results; return true; } @@ -326,7 +369,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B } LDAPSimpleAuthClient client(params.value()); - const auto result = client.authenticate(search_params, search_results); + const auto result = client.authenticate(role_search_params, role_search_results); const auto current_check_timestamp = std::chrono::steady_clock::now(); // Update the cache, but only if this is the latest check and the server is still configured in a compatible way. @@ -345,9 +388,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B std::size_t new_params_hash = 0; new_params.combineCoreHash(new_params_hash); - if (search_params) + if (role_search_params) { - for (const auto & params_instance : *search_params) + for (const auto & params_instance : *role_search_params) { params_instance.combineHash(new_params_hash); } @@ -363,17 +406,17 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B entry.last_successful_params_hash = params_hash; entry.last_successful_authentication_timestamp = current_check_timestamp; - if (search_results) - entry.last_successful_search_results = *search_results; + if (role_search_results) + entry.last_successful_role_search_results = *role_search_results; else - entry.last_successful_search_results.clear(); + entry.last_successful_role_search_results.clear(); } else if ( entry.last_successful_params_hash != params_hash || ( - search_params == nullptr ? - !entry.last_successful_search_results.empty() : - search_params->size() != entry.last_successful_search_results.size() + role_search_params == nullptr ? + !entry.last_successful_role_search_results.empty() : + role_search_params->size() != entry.last_successful_role_search_results.size() ) ) { diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index c8feea7eada..24f1f7b6528 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -34,7 +34,7 @@ public: // The name and readiness of the credentials must be verified before calling these. bool checkLDAPCredentials(const String & server, const BasicCredentials & credentials, - const LDAPClient::SearchParamsList * search_params = nullptr, LDAPClient::SearchResultsList * search_results = nullptr) const; + const LDAPClient::RoleSearchParamsList * role_search_params = nullptr, LDAPClient::SearchResultsList * role_search_results = nullptr) const; bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; GSSAcceptorContext::Params getKerberosParams() const; @@ -44,7 +44,7 @@ private: { std::size_t last_successful_params_hash = 0; std::chrono::steady_clock::time_point last_successful_authentication_timestamp; - LDAPClient::SearchResultsList last_successful_search_results; + LDAPClient::SearchResultsList last_successful_role_search_results; }; using LDAPCache = std::unordered_map; // user name -> cache entry @@ -58,4 +58,6 @@ private: std::optional kerberos_params; }; +void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix); + } diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index b47a9b3e041..c1d54e8c9aa 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -68,34 +68,15 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m common_roles_cfg.insert(role_names.begin(), role_names.end()); } - LDAPClient::SearchParamsList role_search_params_cfg; + LDAPClient::RoleSearchParamsList role_search_params_cfg; if (has_role_mapping) { Poco::Util::AbstractConfiguration::Keys all_keys; config.keys(prefix, all_keys); for (const auto & key : all_keys) { - if (key != "role_mapping" && key.find("role_mapping[") != 0) - continue; - - const String rm_prefix = prefix_str + key; - const String rm_prefix_str = rm_prefix + '.'; - role_search_params_cfg.emplace_back(); - auto & rm_params = role_search_params_cfg.back(); - - rm_params.base_dn = config.getString(rm_prefix_str + "base_dn", ""); - rm_params.search_filter = config.getString(rm_prefix_str + "search_filter", ""); - rm_params.attribute = config.getString(rm_prefix_str + "attribute", "cn"); - rm_params.prefix = config.getString(rm_prefix_str + "prefix", ""); - - auto scope = config.getString(rm_prefix_str + "scope", "subtree"); - boost::algorithm::to_lower(scope); - if (scope == "base") rm_params.scope = LDAPClient::SearchParams::Scope::BASE; - else if (scope == "one_level") rm_params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL; - else if (scope == "subtree") rm_params.scope = LDAPClient::SearchParams::Scope::SUBTREE; - else if (scope == "children") rm_params.scope = LDAPClient::SearchParams::Scope::CHILDREN; - else - throw Exception("Invalid value of 'scope' field in '" + key + "' section of LDAP user directory, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS); + if (key == "role_mapping" || key.find("role_mapping[") == 0) + parseLDAPRoleSearchParams(role_search_params_cfg.emplace_back(), config, prefix_str + key); } } @@ -364,7 +345,7 @@ std::set LDAPAccessStorage::mapExternalRolesNoLock(const LDAPClient::Sea bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials, - const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const + const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const { if (!credentials.isReady()) return false; @@ -373,7 +354,7 @@ bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const C return false; if (const auto * basic_credentials = dynamic_cast(&credentials)) - return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &search_results); + return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &role_search_results); return false; } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index ea0ab47c225..33ac9f0a914 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -68,12 +68,12 @@ private: void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const; std::set mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const; bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials, - const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const; + const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const; mutable std::recursive_mutex mutex; AccessControlManager * access_control_manager = nullptr; String ldap_server_name; - LDAPClient::SearchParamsList role_search_params; + LDAPClient::RoleSearchParamsList role_search_params; std::set common_role_names; // role name that should be granted to all users at all times mutable std::map external_role_hashes; // user name -> LDAPClient::SearchResultsList hash (most recently retrieved and processed) mutable std::map> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles) diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index 5c4b7dd8d99..78b0b7f545b 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -32,6 +32,11 @@ void LDAPClient::SearchParams::combineHash(std::size_t & seed) const boost::hash_combine(seed, static_cast(scope)); boost::hash_combine(seed, search_filter); boost::hash_combine(seed, attribute); +} + +void LDAPClient::RoleSearchParams::combineHash(std::size_t & seed) const +{ + SearchParams::combineHash(seed); boost::hash_combine(seed, prefix); } @@ -42,6 +47,9 @@ void LDAPClient::Params::combineCoreHash(std::size_t & seed) const boost::hash_combine(seed, bind_dn); boost::hash_combine(seed, user); boost::hash_combine(seed, password); + + if (user_dn_detection) + user_dn_detection->combineHash(seed); } LDAPClient::LDAPClient(const Params & params_) @@ -286,18 +294,33 @@ void LDAPClient::openConnection() if (params.enable_tls == LDAPClient::Params::TLSEnable::YES_STARTTLS) diag(ldap_start_tls_s(handle, nullptr, nullptr)); + final_user_name = escapeForLDAP(params.user); + final_bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", final_user_name} }); + final_user_dn = final_bind_dn; // The default value... may be updated right after a successful bind. + switch (params.sasl_mechanism) { case LDAPClient::Params::SASLMechanism::SIMPLE: { - const auto escaped_user_name = escapeForLDAP(params.user); - const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} }); - ::berval cred; cred.bv_val = const_cast(params.password.c_str()); cred.bv_len = params.password.size(); - diag(ldap_sasl_bind_s(handle, bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr)); + diag(ldap_sasl_bind_s(handle, final_bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr)); + + // Once bound, run the user DN search query and update the default value, if asked. + if (params.user_dn_detection) + { + const auto user_dn_search_results = search(*params.user_dn_detection); + + if (user_dn_search_results.size() == 0) + throw Exception("Failed to detect user DN: empty search results", ErrorCodes::LDAP_ERROR); + + if (user_dn_search_results.size() > 1) + throw Exception("Failed to detect user DN: more than one entry in the search results", ErrorCodes::LDAP_ERROR); + + final_user_dn = *user_dn_search_results.begin(); + } break; } @@ -316,6 +339,9 @@ void LDAPClient::closeConnection() noexcept ldap_unbind_ext_s(handle, nullptr, nullptr); handle = nullptr; + final_user_name.clear(); + final_bind_dn.clear(); + final_user_dn.clear(); } LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) @@ -333,10 +359,19 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) case SearchParams::Scope::CHILDREN: scope = LDAP_SCOPE_CHILDREN; break; } - const auto escaped_user_name = escapeForLDAP(params.user); - const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} }); - const auto base_dn = replacePlaceholders(search_params.base_dn, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn} }); - const auto search_filter = replacePlaceholders(search_params.search_filter, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn}, {"{base_dn}", base_dn} }); + const auto final_base_dn = replacePlaceholders(search_params.base_dn, { + {"{user_name}", final_user_name}, + {"{bind_dn}", final_bind_dn}, + {"{user_dn}", final_user_dn} + }); + + const auto final_search_filter = replacePlaceholders(search_params.search_filter, { + {"{user_name}", final_user_name}, + {"{bind_dn}", final_bind_dn}, + {"{user_dn}", final_user_dn}, + {"{base_dn}", final_base_dn} + }); + char * attrs[] = { const_cast(search_params.attribute.c_str()), nullptr }; ::timeval timeout = { params.search_timeout.count(), 0 }; LDAPMessage* msgs = nullptr; @@ -349,7 +384,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) } }); - diag(ldap_search_ext_s(handle, base_dn.c_str(), scope, search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs)); + diag(ldap_search_ext_s(handle, final_base_dn.c_str(), scope, final_search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs)); for ( auto * msg = ldap_first_message(handle, msgs); @@ -361,6 +396,27 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) { case LDAP_RES_SEARCH_ENTRY: { + // Extract DN separately, if the requested attribute is DN. + if (boost::iequals("dn", search_params.attribute)) + { + BerElement * ber = nullptr; + + SCOPE_EXIT({ + if (ber) + { + ber_free(ber, 0); + ber = nullptr; + } + }); + + ::berval bv; + + diag(ldap_get_dn_ber(handle, msg, &ber, &bv)); + + if (bv.bv_val && bv.bv_len > 0) + result.emplace(bv.bv_val, bv.bv_len); + } + BerElement * ber = nullptr; SCOPE_EXIT({ @@ -471,12 +527,12 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) return result; } -bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, SearchResultsList * search_results) +bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results) { if (params.user.empty()) throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); - if (!search_params != !search_results) + if (!role_search_params != !role_search_results) throw Exception("Cannot return LDAP search results", ErrorCodes::BAD_ARGUMENTS); // Silently reject authentication attempt if the password is empty as if it didn't match. @@ -489,21 +545,21 @@ bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, openConnection(); // While connected, run search queries and save the results, if asked. - if (search_params) + if (role_search_params) { - search_results->clear(); - search_results->reserve(search_params->size()); + role_search_results->clear(); + role_search_results->reserve(role_search_params->size()); try { - for (const auto & single_search_params : *search_params) + for (const auto & params_instance : *role_search_params) { - search_results->emplace_back(search(single_search_params)); + role_search_results->emplace_back(search(params_instance)); } } catch (...) { - search_results->clear(); + role_search_results->clear(); throw; } } @@ -532,7 +588,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams &) throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } -bool LDAPSimpleAuthClient::authenticate(const SearchParamsList *, SearchResultsList *) +bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList *, SearchResultsList *) { throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index 4fc97bb957b..388e7ad0f0d 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -38,12 +38,20 @@ public: Scope scope = Scope::SUBTREE; String search_filter; String attribute = "cn"; + + void combineHash(std::size_t & seed) const; + }; + + struct RoleSearchParams + : public SearchParams + { String prefix; void combineHash(std::size_t & seed) const; }; - using SearchParamsList = std::vector; + using RoleSearchParamsList = std::vector; + using SearchResults = std::set; using SearchResultsList = std::vector; @@ -105,6 +113,8 @@ public: String user; String password; + std::optional user_dn_detection; + std::chrono::seconds verification_cooldown{0}; std::chrono::seconds operation_timeout{40}; @@ -134,6 +144,9 @@ protected: #if USE_LDAP LDAP * handle = nullptr; #endif + String final_user_name; + String final_bind_dn; + String final_user_dn; }; class LDAPSimpleAuthClient @@ -141,7 +154,7 @@ class LDAPSimpleAuthClient { public: using LDAPClient::LDAPClient; - bool authenticate(const SearchParamsList * search_params, SearchResultsList * search_results); + bool authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results); }; } From a9e5532da62873ae7d9920086ca83aaae161df43 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 29 Mar 2021 12:27:16 +0400 Subject: [PATCH 002/202] Fix builds: soothe the linters --- src/Access/ExternalAuthenticators.cpp | 5 ++++- src/Access/LDAPClient.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 9eaf2a4b04b..99a3347b0de 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -99,7 +99,10 @@ void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConf if (has_user_dn_detection) { if (!params.user_dn_detection) - params.user_dn_detection = { .attribute = "dn" }; + { + params.user_dn_detection.emplace(); + params.user_dn_detection->attribute = "dn"; + } parseLDAPSearchParams(*params.user_dn_detection, config, ldap_server_config + ".user_dn_detection"); } diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index 78b0b7f545b..a8f9675774b 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -313,7 +313,7 @@ void LDAPClient::openConnection() { const auto user_dn_search_results = search(*params.user_dn_detection); - if (user_dn_search_results.size() == 0) + if (user_dn_search_results.empty()) throw Exception("Failed to detect user DN: empty search results", ErrorCodes::LDAP_ERROR); if (user_dn_search_results.size() > 1) From 0e5c58c8b20d56eded6e2b786f8ae97ebd2ae466 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 19 Apr 2021 19:00:30 -0400 Subject: [PATCH 003/202] Adding user DN detection tests. --- .../testflows/ldap/role_mapping/regression.py | 3 + .../role_mapping/requirements/requirements.md | 70 ++- .../role_mapping/requirements/requirements.py | 184 ++++++- .../ldap/role_mapping/tests/common.py | 7 +- .../role_mapping/tests/user_dn_detection.py | 474 ++++++++++++++++++ 5 files changed, 728 insertions(+), 10 deletions(-) create mode 100644 tests/testflows/ldap/role_mapping/tests/user_dn_detection.py diff --git a/tests/testflows/ldap/role_mapping/regression.py b/tests/testflows/ldap/role_mapping/regression.py index 7afb6c98713..c853316ecec 100755 --- a/tests/testflows/ldap/role_mapping/regression.py +++ b/tests/testflows/ldap/role_mapping/regression.py @@ -11,6 +11,8 @@ from ldap.role_mapping.requirements import * # Cross-outs of known fails xfails = { "mapping/roles removed and added in parallel": + [(Fail, "known bug")], + "user dn detection/mapping/roles removed and added in parallel": [(Fail, "known bug")] } @@ -42,6 +44,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): Scenario(run=load("ldap.authentication.tests.sanity", "scenario"), name="ldap sanity") Feature(run=load("ldap.role_mapping.tests.server_config", "feature")) Feature(run=load("ldap.role_mapping.tests.mapping", "feature")) + Feature(run=load("ldap.role_mapping.tests.user_dn_detection", "feature")) if main(): regression() diff --git a/tests/testflows/ldap/role_mapping/requirements/requirements.md b/tests/testflows/ldap/role_mapping/requirements/requirements.md index e79baa9cd7c..fbd772b9d29 100644 --- a/tests/testflows/ldap/role_mapping/requirements/requirements.md +++ b/tests/testflows/ldap/role_mapping/requirements/requirements.md @@ -44,6 +44,11 @@ * 4.7.1 [BindDN Parameter](#binddn-parameter) * 4.7.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN](#rqsrs-014ldaprolemappingconfigurationserverbinddn) * 4.7.1.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN](#rqsrs-014ldaprolemappingconfigurationserverbinddnconflictwithauthdn) + * 4.7.2 [User DN Detection](#user-dn-detection) + * 4.7.2.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection](#rqsrs-014ldaprolemappingconfigurationserveruserdndetection) + * 4.7.2.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionbasedn) + * 4.7.2.3 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionscope) + * 4.7.2.4 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionsearchfilter) * 4.8 [External User Directory Configuration](#external-user-directory-configuration) * 4.8.1 [Syntax](#syntax) * 4.8.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingsyntax) @@ -318,6 +323,67 @@ version: 1.0 [ClickHouse] SHALL return an error if both `` and `` or `` parameters are specified as part of [LDAP] server description in the `` section of the `config.xml`. +#### User DN Detection + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection +version: 1.0 + +[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section +of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN +version: 1.0 + +[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify how +to construct the base DN for the LDAP search to detect the actual user DN. + +For example, + +```xml + + ... + CN=Users,DC=example,DC=com + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope +version: 1.0 + +[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL the scope of the +LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values + +* `base` +* `one_level` +* `children` +* `subtree` + +For example, + +```xml + + ... + one_level + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter +version: 1.0 + +[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify the LDAP search +filter used to detect the actual user DN. + +For example, + +```xml + + ... + (&(objectClass=user)(sAMAccountName={user_name})) + +``` + ### External User Directory Configuration #### Syntax @@ -382,7 +448,7 @@ version: 1.0 [ClickHouse] SHALL support the `` parameter in the `` section of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search. -The resulting `DN` SHALL be constructed by replacing all the `{user_name}` and `{bind_dn}` substrings of +The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of the template with the actual user name and bind `DN` during each [LDAP] search. #### Attribute Parameter @@ -445,7 +511,7 @@ version: 1.0 section of the `config.xml` that SHALL specify the template used to construct the [LDAP filter](https://ldap.com/ldap-filters/) for the search. -The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings +The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search. #### Prefix Parameter diff --git a/tests/testflows/ldap/role_mapping/requirements/requirements.py b/tests/testflows/ldap/role_mapping/requirements/requirements.py index b2748762e03..68ce4f5913e 100644 --- a/tests/testflows/ldap/role_mapping/requirements/requirements.py +++ b/tests/testflows/ldap/role_mapping/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.210129.1222545. +# document by TestFlows v1.6.210412.1213859. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -488,6 +488,105 @@ RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN_ConflictWith_AuthDN = Re level=4, num='4.7.1.2') +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section\n' + 'of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. \n' + '\n' + ), + link=None, + level=4, + num='4.7.2.1') + +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the \n' + '`` section of the `config.xml` that SHALL specify how \n' + 'to construct the base DN for the LDAP search to detect the actual user DN.\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + '\n' + ' ...\n' + ' CN=Users,DC=example,DC=com\n' + '\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='4.7.2.2') + +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the \n' + '`` section of the `config.xml` that SHALL the scope of the \n' + 'LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values\n' + '\n' + '* `base`\n' + '* `one_level`\n' + '* `children`\n' + '* `subtree`\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + '\n' + ' ...\n' + ' one_level\n' + '\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='4.7.2.3') + +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the \n' + '`` section of the `config.xml` that SHALL specify the LDAP search\n' + 'filter used to detect the actual user DN.\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + '\n' + ' ...\n' + ' (&(objectClass=user)(sAMAccountName={user_name}))\n' + '\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='4.7.2.4') + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Syntax = Requirement( name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax', version='1.0', @@ -587,7 +686,7 @@ RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_BaseDN = Req '[ClickHouse] SHALL support the `` parameter in the `` section \n' 'of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search.\n' '\n' - 'The resulting `DN` SHALL be constructed by replacing all the `{user_name}` and `{bind_dn}` substrings of \n' + 'The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of \n' 'the template with the actual user name and bind `DN` during each [LDAP] search.\n' '\n' ), @@ -724,7 +823,7 @@ RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SearchFilter 'section of the `config.xml` that SHALL specify the template used to construct \n' 'the [LDAP filter](https://ldap.com/ldap-filters/) for the search.\n' '\n' - 'The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings \n' + 'The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings \n' 'of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search.\n' ' \n' ), @@ -872,6 +971,11 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( Heading(name='BindDN Parameter', level=3, num='4.7.1'), Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN', level=4, num='4.7.1.1'), Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN', level=4, num='4.7.1.2'), + Heading(name='User DN Detection', level=3, num='4.7.2'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection', level=4, num='4.7.2.1'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN', level=4, num='4.7.2.2'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope', level=4, num='4.7.2.3'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter', level=4, num='4.7.2.4'), Heading(name='External User Directory Configuration', level=2, num='4.8'), Heading(name='Syntax', level=3, num='4.8.1'), Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax', level=4, num='4.8.1.1'), @@ -930,6 +1034,10 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_SameUser, RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN, RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN_ConflictWith_AuthDN, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Syntax, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SpecialCharactersEscaping, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections, @@ -996,6 +1104,11 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( * 4.7.1 [BindDN Parameter](#binddn-parameter) * 4.7.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN](#rqsrs-014ldaprolemappingconfigurationserverbinddn) * 4.7.1.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN](#rqsrs-014ldaprolemappingconfigurationserverbinddnconflictwithauthdn) + * 4.7.2 [User DN Detection](#user-dn-detection) + * 4.7.2.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection](#rqsrs-014ldaprolemappingconfigurationserveruserdndetection) + * 4.7.2.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionbasedn) + * 4.7.2.3 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionscope) + * 4.7.2.4 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionsearchfilter) * 4.8 [External User Directory Configuration](#external-user-directory-configuration) * 4.8.1 [Syntax](#syntax) * 4.8.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingsyntax) @@ -1270,6 +1383,67 @@ version: 1.0 [ClickHouse] SHALL return an error if both `` and `` or `` parameters are specified as part of [LDAP] server description in the `` section of the `config.xml`. +#### User DN Detection + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection +version: 1.0 + +[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section +of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN +version: 1.0 + +[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify how +to construct the base DN for the LDAP search to detect the actual user DN. + +For example, + +```xml + + ... + CN=Users,DC=example,DC=com + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope +version: 1.0 + +[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL the scope of the +LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values + +* `base` +* `one_level` +* `children` +* `subtree` + +For example, + +```xml + + ... + one_level + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter +version: 1.0 + +[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify the LDAP search +filter used to detect the actual user DN. + +For example, + +```xml + + ... + (&(objectClass=user)(sAMAccountName={user_name})) + +``` + ### External User Directory Configuration #### Syntax @@ -1334,7 +1508,7 @@ version: 1.0 [ClickHouse] SHALL support the `` parameter in the `` section of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search. -The resulting `DN` SHALL be constructed by replacing all the `{user_name}` and `{bind_dn}` substrings of +The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of the template with the actual user name and bind `DN` during each [LDAP] search. #### Attribute Parameter @@ -1397,7 +1571,7 @@ version: 1.0 section of the `config.xml` that SHALL specify the template used to construct the [LDAP filter](https://ldap.com/ldap-filters/) for the search. -The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings +The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search. #### Prefix Parameter diff --git a/tests/testflows/ldap/role_mapping/tests/common.py b/tests/testflows/ldap/role_mapping/tests/common.py index 33ad4a46f52..565503296e3 100644 --- a/tests/testflows/ldap/role_mapping/tests/common.py +++ b/tests/testflows/ldap/role_mapping/tests/common.py @@ -24,11 +24,12 @@ def create_table(self, name, create_statement, on_cluster=False): node.query(f"DROP TABLE IF EXISTS {name}") @TestStep(Given) -def add_ldap_servers_configuration(self, servers, config_d_dir="/etc/clickhouse-server/config.d", +def add_ldap_servers_configuration(self, servers, config=None, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", timeout=60, restart=False): """Add LDAP servers configuration to config.xml. """ - config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + if config is None: + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) return add_config(config, restart=restart) @TestStep(Given) @@ -249,4 +250,4 @@ def create_ldap_external_user_directory_config_content(server=None, roles=None, def create_entries_ldap_external_user_directory_config_content(entries, **kwargs): """Create LDAP external user directory configuration file content. """ - return create_xml_config_content(entries, **kwargs) \ No newline at end of file + return create_xml_config_content(entries, **kwargs) diff --git a/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py new file mode 100644 index 00000000000..9ec24040973 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py @@ -0,0 +1,474 @@ +# -*- coding: utf-8 -*- +import importlib + +from testflows.core import * +from testflows.asserts import error + +from ldap.role_mapping.requirements import * +from ldap.role_mapping.tests.common import * + +@TestOutline +def check_config(self, entries, valid=True, ldap_server="openldap1", user="user1", password="user1"): + """Apply LDAP server configuration and check login. + """ + if valid: + exitcode = 0 + message = "1" + else: + exitcode = 4 + message = "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + + with Given("I add LDAP server configuration"): + config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + add_ldap_servers_configuration(servers=None, config=config) + + with And("I add LDAP external user directory configuration"): + add_ldap_external_user_directory(server=ldap_server, + role_mappings=None, restart=True) + + with When(f"I login I try to login as an LDAP user"): + r = self.context.node.query(f"SELECT 1", settings=[ + ("user", user), ("password", password)], exitcode=exitcode, message=message) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") +) +def config_invalid_base_dn(self): + """Check when invalid `base_dn` is specified in the user_dn_detection section. + """ + + with Given("I define LDAP server configuration with invalid base_dn"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=user,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") +) +def config_empty_base_dn(self): + """Check when empty `base_dn` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid base_dn"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") +) +def config_missing_base_dn(self): + """Check when missing `base_dn` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid base_dn"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + # FIXME +) +def config_invalid_search_filter(self): + """Check when invalid `search_filter` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid search_filter"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPersons)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_missing_search_filter(self): + """Check when missing `search_filter` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid search_filter"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_empty_search_filter(self): + """Check when empty `search_filter` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid search_filter"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"), + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_valid(self): + """Check valid config with valid user_dn_detection section. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=True) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"), + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_valid_tls_connection(self): + """Check valid config with valid user_dn_detection section when + using LDAP that is configured to use TLS connection. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "tls_require_cert": "never", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=True, ldap_server="openldap2", user="user2", password="user2") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope("1.0") +) +@Examples("scope base_dn", [ + ("base", "cn=user1,ou=users,dc=company,dc=com"), + ("one_level","ou=users,dc=company,dc=com"), + ("children","ou=users,dc=company,dc=com"), + ("subtree","ou=users,dc=company,dc=com") # default value +]) +def check_valid_scope_values(self, scope, base_dn): + """Check configuration with valid scope values. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": base_dn, + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + "scope": scope + } + } + } + ] + } + + check_config(entries=entries, valid=True) + +@TestSuite +def mapping(self): + """Run all role mapping tests with both + openldap1 and openldap2 configured to use + user DN detection. + """ + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True, + "dn": "cn=user1,ou=users,dc=company,dc=com"}, + ] + + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + }, + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "tls_require_cert": "never", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + }, + ] + } + + with Given("I add LDAP servers configuration"): + config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + add_ldap_servers_configuration(servers=None, config=config) + + for scenario in loads(importlib.import_module("tests.mapping", package=None), Scenario): + scenario(ldap_server="openldap1", ldap_user=users[0]) + +@TestOutline +def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection): + """Check that roles get mapped properly when bind_dn and user_dn are different + by creating LDAP users that have switched uid parameter values. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + } + } + ] + } + + if user_dn_detection: + with And("I enable user dn detection"): + entries["ldap_servers"][0]["openldap1"]["user_dn_detection"] = { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + "scope": "subtree" + } + + with And("I define role mappings"): + role_mappings = [ + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": f"(&(objectClass=groupOfUniqueNames)(uniquemember={{{map_by}}}))", + "prefix":"" + } + ] + + with Given("I add LDAP users"): + first_user = add_ldap_users(users=[ + {"cn": f"first_user", "userpassword": "user", "uid": "second_user"} + ])[0] + + second_user = add_ldap_users(users=[ + {"cn": f"second_user", "userpassword": "user", "uid": "first_user"} + ])[0] + + with Given("I add LDAP groups"): + groups = add_ldap_groups(groups=({"cn": f"role0_{uid}"}, {"cn": f"role1_{uid}"})) + + with And("I add LDAP user to each LDAP group"): + with By("adding first group to first user"): + add_user_to_group_in_ldap(user=first_user, group=groups[0]) + with And("adding second group to second user"): + add_user_to_group_in_ldap(user=second_user, group=groups[1]) + + with And("I add RBAC roles"): + roles = add_rbac_roles(roles=(f"role0_{uid}", f"role1_{uid}")) + + with Given("I add LDAP server configuration"): + config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + add_ldap_servers_configuration(servers=None, config=config) + + with And("I add LDAP external user directory configuration"): + add_ldap_external_user_directory(server=self.context.ldap_node.name, + role_mappings=role_mappings, restart=True) + +@TestScenario +def map_roles_by_user_dn_when_base_dn_and_user_dn_are_different(self): + """Check the case when we map roles using user_dn then + the first user has uid of second user and second user + has uid of first user and configuring user DN detection to + determine user_dn based on the uid value so that user_dn + for the first user will be bind_dn of the second user and + vice versa. + """ + uid = getuid() + + setup_different_bind_dn_and_user_dn(uid=uid, map_by="user_dn", user_dn_detection=True) + + with When(f"I login as first LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "first_user"), ("password", "user")]) + + with Then("I expect the first user to have mapped LDAP roles from second user"): + assert f"GRANT role1_{uid} TO first_user" in r.output, error() + + with When(f"I login as second LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "second_user"), ("password", "user")]) + + with Then("I expect the second user to have mapped LDAP roles from first user"): + assert f"GRANT role0_{uid} TO second_user" in r.output, error() + +@TestScenario +def map_roles_by_bind_dn_when_base_dn_and_user_dn_are_different(self): + """Check the case when we map roles by bind_dn when bind_dn and user_dn + are different. + """ + uid = getuid() + + setup_different_bind_dn_and_user_dn(uid=uid, map_by="bind_dn", user_dn_detection=True) + + with When(f"I login as first LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "first_user"), ("password", "user")]) + + with Then("I expect the first user to have no mapped LDAP roles"): + assert f"GRANT role0_{uid} TO first_user" == r.output, error() + + with When(f"I login as second LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "second_user"), ("password", "user")]) + + with Then("I expect the second user to have no mapped LDAP roles"): + assert f"GRANT role1_{uid} TO second_user" in r.output, error() + +@TestFeature +@Name("user dn detection") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection("1.0") +) +def feature(self): + """Check LDAP user DN detection. + """ + self.context.node = self.context.cluster.node("clickhouse1") + self.context.ldap_node = self.context.cluster.node("openldap1") + + with Given("I fix LDAP access permissions"): + fix_ldap_permissions(node=self.context.cluster.node("openldap1")) + fix_ldap_permissions(node=self.context.cluster.node("openldap2")) + + for scenario in ordered(loads(current_module(), Scenario)): + scenario() + + Suite(run=mapping) From 47a4ae930f59fa888ec86f187ad22e6ace350cb9 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Mon, 3 May 2021 23:44:46 +0300 Subject: [PATCH 004/202] edited --- .../parametric-functions.md | 2 +- .../functions/ip-address-functions.md | 4 +- .../functions/type-conversion-functions.md | 48 +++++++++++--- .../parametric-functions.md | 2 +- .../functions/ip-address-functions.md | 8 +-- .../functions/type-conversion-functions.md | 66 ++++++++++++++----- 6 files changed, 97 insertions(+), 33 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index b9d504241db..83644ef1272 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -253,7 +253,7 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN) **Parameters** -- `window` — Length of the sliding window, it is the time interval between first condition and last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`. +- `window` — Length of the sliding window, it is the time interval between the first condition and last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`. - `mode` — It is an optional argument. One or more modes can be set. - `'strict'` — If same condition holds for sequence of events then such non-unique events would be skipped. - `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2. diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md index 0b5dd7160b8..d37ef2e8f1a 100644 --- a/docs/en/sql-reference/functions/ip-address-functions.md +++ b/docs/en/sql-reference/functions/ip-address-functions.md @@ -422,7 +422,7 @@ Type: [UInt8](../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8') +SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8'); ``` Result: @@ -436,7 +436,7 @@ Result: Query: ``` sql -SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16') +SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16'); ``` Result: diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index d8d13d81d97..0d9053c993d 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -373,7 +373,7 @@ This function accepts a number or date or date with time, and returns a FixedStr ## reinterpretAsUUID {#reinterpretasuuid} -This function accepts 16 bytes string, and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the functions work as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored. +Accepts 16 bytes string and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the functions works as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored. **Syntax** @@ -429,7 +429,24 @@ Result: ## reinterpret(x, T) {#type_conversion_function-reinterpret} -Use the same source in-memory bytes sequence for `x` value and reinterpret it to destination type +Uses the same source in-memory bytes sequence for `x` value and reinterprets it to destination type. + +**Syntax** + +``` sql +reinterpret(x, type) +``` + +**Arguments** + +- `x` — Any type. +- `type` — Destination type. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- Destination type value. + +**Examples** Query: ```sql @@ -448,21 +465,36 @@ Result: ## CAST(x, T) {#type_conversion_function-cast} -Converts input value `x` to the `T` data type. Unlike to `reinterpret` function use external representation of `x` value. +Converts input value `x` to the `T` data type. Unlike to `reinterpret` function, type conversion is performed in a natural way. The syntax `CAST(x AS t)` is also supported. Note, that if value `x` does not fit the bounds of type T, the function overflows. For example, CAST(-1, 'UInt8') returns 255. +**Syntax** + +``` sql +CAST(x, T) +``` + +**Arguments** + +- `x` — Any type. +- `T` — Destination type. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- Destination type value. + **Examples** Query: ```sql SELECT - cast(toInt8(-1), 'UInt8') AS cast_int_to_uint, - cast(toInt8(1), 'Float32') AS cast_int_to_float, - cast('1', 'UInt32') AS cast_string_to_int + CAST(toInt8(-1), 'UInt8') AS cast_int_to_uint, + CAST(toInt8(1), 'Float32') AS cast_int_to_float, + CAST('1', 'UInt32') AS cast_string_to_int ``` Result: @@ -492,7 +524,7 @@ Result: └─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ ``` -Conversion to FixedString(N) only works for arguments of type String or FixedString(N). +Conversion to FixedString(N) only works for arguments of type [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md). Type conversion to [Nullable](../../sql-reference/data-types/nullable.md) and back is supported. @@ -1038,7 +1070,7 @@ Result: ## parseDateTime64BestEffort {#parsedatetime64besteffort} -Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and return `DateTime64(3)` or `DateTime64(6)` data types. +Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and returns `DateTime64(3)` or `DateTime64(6)` data types. **Syntax** diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index e5162b63b88..90754aa999b 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -253,7 +253,7 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN) **Параметры** -- `window` — ширина скользящего окна по времени. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond2 <= timestamp события cond1 + window`. +- `window` — ширина скользящего окна по времени. Это время между первым и последним условием. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond1 <= timestamp события cond2 <= ... <= timestamp события condN <= timestamp события cond1 + window`. - `mode` — необязательный параметр. Может быть установленно несколько значений одновременно. - `'strict'` — не учитывать подряд идущие повторяющиеся события. - `'strict_order'` — запрещает посторонние события в искомой последовательности. Например, при поиске цепочки `A->B->C` в `A->B->D->C` поиск будет остановлен на `D` и функция вернет 2. diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index d7f6d2f7618..10ded819fef 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -397,7 +397,7 @@ SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0 ## isIPAddressInRange {#isipaddressinrange} -Проверяет попадает ли IP адрес в интервал, заданный в [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) нотации. +Проверяет, попадает ли IP адрес в интервал, заданный в нотации [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). **Syntax** @@ -409,7 +409,7 @@ isIPAddressInRange(address, prefix) **Аргументы** - `address` — IPv4 или IPv6 адрес. [String](../../sql-reference/data-types/string.md). -- `prefix` — IPv4 или IPv6 подсеть, заданная в CIDR нотации. [String](../../sql-reference/data-types/string.md). +- `prefix` — IPv4 или IPv6 подсеть, заданная в нотации CIDR. [String](../../sql-reference/data-types/string.md). **Возвращаемое значение** @@ -422,7 +422,7 @@ isIPAddressInRange(address, prefix) Запрос: ``` sql -SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8') +SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8'); ``` Результат: @@ -436,7 +436,7 @@ SELECT isIPAddressInRange('127.0.0.1', '127.0.0.0/8') Запрос: ``` sql -SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16') +SELECT isIPAddressInRange('127.0.0.1', 'ffff::/16'); ``` Результат: diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index fc1dd15f8e3..5f79ac635ba 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -369,7 +369,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut; ## reinterpretAsUUID {#reinterpretasuuid} -Функция принимает шестнадцатибайтную строку и интерпретирует ее байты в network order (big-endian). Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количетсвом нулевых байт с конца. Если строка длиннее, чем шестнадцать байт, то игнорируются лишние байты с конца. +Функция принимает шестнадцатибайтную строку и интерпретирует ее байты в network order (big-endian). Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количеством нулевых байтов с конца. Если строка длиннее, чем шестнадцать байтов, то игнорируются лишние байты с конца. **Синтаксис** @@ -425,9 +425,27 @@ SELECT uuid = uuid2; ## reinterpret(x, T) {#type_conversion_function-reinterpret} -Использует туже самую исходную последовательность байт в памяти для значения `x` и переинтерпретирует ее как конечный тип данных +Использует ту же самую исходную последовательность байтов в памяти для значения `x` и интерпретирует ее как конечный тип данных `T`. + +**Синтаксис** + +``` sql +reinterpret(x, type) +``` + +**Аргументы** + +- `x` — любой тип данных. +- `type` — конечный тип данных. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Конечный тип данных. + +**Примеры** Запрос: + ```sql SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint, reinterpret(toInt8(1), 'Float32') as int_to_float, @@ -450,15 +468,30 @@ SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint, Обратите внимание, что если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255. +**Синтаксис** + +``` sql +CAST(x, T) +``` + +**Аргументы** + +- `x` — любой тип данных. +- `T` — конечный тип данных. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- Конечный тип данных. + **Примеры** Запрос: ```sql SELECT - cast(toInt8(-1), 'UInt8') AS cast_int_to_uint, - cast(toInt8(1), 'Float32') AS cast_int_to_float, - cast('1', 'UInt32') AS cast_string_to_int + CAST(toInt8(-1), 'UInt8') AS cast_int_to_uint, + CAST(toInt8(1), 'Float32') AS cast_int_to_float, + CAST('1', 'UInt32') AS cast_string_to_int ``` Результат: @@ -488,7 +521,7 @@ SELECT └─────────────────────┴─────────────────────┴────────────┴─────────────────────┴───────────────────────────┘ ``` -Преобразование в FixedString(N) работает только для аргументов типа String или FixedString(N). +Преобразование в FixedString(N) работает только для аргументов типа [String](../../sql-reference/data-types/string.md) или [FixedString](../../sql-reference/data-types/fixedstring.md). Поддержано преобразование к типу [Nullable](../../sql-reference/functions/type-conversion-functions.md) и обратно. @@ -860,7 +893,7 @@ AS parseDateTimeBestEffortUS; ## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero} ## parseDateTime32BestEffortOrZero {#parsedatetime32besteffortorzero} -Работает также как [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевую дату или нулевую дату и время когда получает формат даты который не может быть обработан. +Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевую дату или нулевую дату и время, когда получает формат даты, который не может быть обработан. ## parseDateTimeBestEffortUSOrNull {#parsedatetimebesteffortusornull} @@ -1036,19 +1069,19 @@ SELECT parseDateTimeBestEffortUSOrZero('02.2021') AS parseDateTimeBestEffortUSOr ## parseDateTime64BestEffort {#parsedatetime64besteffort} -Работает также как функция [parseDateTimeBestEffort](#parsedatetimebesteffort) но также понимамет милисекунды и микросекунды и возвращает `DateTime64(3)` или `DateTime64(6)` типы данных в зависимости от заданной точности. +Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но также принимает миллисекунды и микросекунды и возвращает типы данных `DateTime64(3)` или `DateTime64(6)` в зависимости от заданной точности. -**Syntax** +**Синтаксис** ``` sql parseDateTime64BestEffort(time_string [, precision [, time_zone]]) ``` -**Parameters** +**Параметры** -- `time_string` — String containing a date or date with time to convert. [String](../../sql-reference/data-types/string.md). -- `precision` — `3` for milliseconds, `6` for microseconds. Default `3`. Optional [UInt8](../../sql-reference/data-types/int-uint.md). -- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). +- `time_string` — строка, содержащая дату или дату со временем, которые нужно преобразовать. [String](../../sql-reference/data-types/string.md). +- `precision` — `3` для миллисекунд, `6` для микросекунд. По умолчанию `3`. Необязательный. [UInt8](../../sql-reference/data-types/int-uint.md). +- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). Разбирает значение `time_string` в зависимости от часового пояса. Необязательный. [String](../../sql-reference/data-types/string.md). **Примеры** @@ -1078,12 +1111,11 @@ FORMAT PrettyCompactMonoBlcok ## parseDateTime64BestEffortOrNull {#parsedatetime32besteffortornull} -Работает также как функция [parseDateTime64BestEffort](#parsedatetime64besteffort) но возвращает `NULL` когда встречает формат даты который не может обработать. +Работает аналогично функции [parseDateTime64BestEffort](#parsedatetime64besteffort), но возвращает `NULL`, когда встречает формат даты, который не может обработать. ## parseDateTime64BestEffortOrZero {#parsedatetime64besteffortorzero} -Работает также как функция [parseDateTime64BestEffort](#parsedatetimebesteffort) но возвращает "нулевую" дату и время когда встречает формат даты который не может обработать. - +Работает аналогично функции [parseDateTime64BestEffort](#parsedatetimebesteffort), но возвращает "нулевую" дату и время, когда встречает формат даты, который не может обработать. ## toLowCardinality {#tolowcardinality} @@ -1130,7 +1162,7 @@ SELECT toLowCardinality('1'); ## toUnixTimestamp64Nano {#tounixtimestamp64nano} Преобразует значение `DateTime64` в значение `Int64` с фиксированной точностью менее одной секунды. -Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что возвращаемое значение - это временная метка в UTC, а не в часовом поясе `DateTime64`. +Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`. **Синтаксис** From 458cd6961ce5d566a202e932f8b1c60d37dd3d49 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Tue, 4 May 2021 13:19:45 +0300 Subject: [PATCH 005/202] minor changes --- .../parametric-functions.md | 2 +- .../functions/type-conversion-functions.md | 18 +++++++++++------- .../parametric-functions.md | 2 +- .../functions/type-conversion-functions.md | 12 ++++++++---- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 83644ef1272..2a221bbb6eb 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -312,7 +312,7 @@ FROM GROUP BY user_id ) GROUP BY level -ORDER BY level ASC +ORDER BY level ASC; ``` Result: diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 0d9053c993d..87dbfa1728f 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -469,7 +469,7 @@ Converts input value `x` to the `T` data type. Unlike to `reinterpret` function, The syntax `CAST(x AS t)` is also supported. -Note, that if value `x` does not fit the bounds of type T, the function overflows. For example, CAST(-1, 'UInt8') returns 255. +Note, that if value `x` does not fit the bounds of type `T`, the function overflows. For example, `CAST(-1, 'UInt8')` returns `255`. **Syntax** @@ -494,7 +494,7 @@ Query: SELECT CAST(toInt8(-1), 'UInt8') AS cast_int_to_uint, CAST(toInt8(1), 'Float32') AS cast_int_to_float, - CAST('1', 'UInt32') AS cast_string_to_int + CAST('1', 'UInt32') AS cast_string_to_int; ``` Result: @@ -1070,7 +1070,7 @@ Result: ## parseDateTime64BestEffort {#parsedatetime64besteffort} -Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and returns `DateTime64(3)` or `DateTime64(6)` data types. +Same as [parseDateTimeBestEffort](#parsedatetimebesteffort) function but also parse milliseconds and microseconds and returns [DateTime](../../sql-reference/functions/type-conversion-functions.md#data_type-datetime) data type. **Syntax** @@ -1081,9 +1081,13 @@ parseDateTime64BestEffort(time_string [, precision [, time_zone]]) **Parameters** - `time_string` — String containing a date or date with time to convert. [String](../../sql-reference/data-types/string.md). -- `precision` — `3` for milliseconds, `6` for microseconds. Default `3`. Optional [UInt8](../../sql-reference/data-types/int-uint.md). +- `precision` — `3` for milliseconds, `6` for microseconds. Default `3`. Optional. [UInt8](../../sql-reference/data-types/int-uint.md). - `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). +**Returned value** + +- `time_string` converted to the [DateTime](../../sql-reference/data-types/datetime.md) data type. + **Examples** Query: @@ -1096,7 +1100,7 @@ UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t -FORMAT PrettyCompactMonoBlcok +FORMAT PrettyCompactMonoBlcok; ``` Result: @@ -1168,7 +1172,7 @@ Input value is scaled up or down appropriately depending on it precision. Please **Syntax** -``` sql +```sql toUnixTimestamp64Milli(value) ``` @@ -1184,7 +1188,7 @@ toUnixTimestamp64Milli(value) Query: -``` sql +```sql WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 SELECT toUnixTimestamp64Milli(dt64); ``` diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 90754aa999b..508c8de2a58 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -311,7 +311,7 @@ FROM GROUP BY user_id ) GROUP BY level -ORDER BY level ASC +ORDER BY level ASC; ``` ## retention {#retention} diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 5f79ac635ba..46032f29551 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1069,7 +1069,7 @@ SELECT parseDateTimeBestEffortUSOrZero('02.2021') AS parseDateTimeBestEffortUSOr ## parseDateTime64BestEffort {#parsedatetime64besteffort} -Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но также принимает миллисекунды и микросекунды и возвращает типы данных `DateTime64(3)` или `DateTime64(6)` в зависимости от заданной точности. +Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но также принимает миллисекунды и микросекунды. Возвращает тип данных [DateTime](../../sql-reference/functions/type-conversion-functions.md#data_type-datetime). **Синтаксис** @@ -1083,6 +1083,10 @@ parseDateTime64BestEffort(time_string [, precision [, time_zone]]) - `precision` — `3` для миллисекунд, `6` для микросекунд. По умолчанию `3`. Необязательный. [UInt8](../../sql-reference/data-types/int-uint.md). - `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). Разбирает значение `time_string` в зависимости от часового пояса. Необязательный. [String](../../sql-reference/data-types/string.md). +**Возвращаемое значение** + +- `time_string`, преобразованная в тип данных [DateTime](../../sql-reference/data-types/datetime.md). + **Примеры** Запрос: @@ -1095,7 +1099,7 @@ UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t -FORMAT PrettyCompactMonoBlcok +FORMAT PrettyCompactMonoBlcok; ``` Результат: @@ -1166,7 +1170,7 @@ SELECT toLowCardinality('1'); **Синтаксис** -``` sql +```sql toUnixTimestamp64Milli(value) ``` @@ -1182,7 +1186,7 @@ toUnixTimestamp64Milli(value) Запрос: -``` sql +```sql WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64 SELECT toUnixTimestamp64Milli(dt64); ``` From 4aad69dc90b56c3f1fc62849f6be2658feec91fb Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 1 May 2021 18:00:43 +0000 Subject: [PATCH 006/202] Better commnets --- src/Storages/RabbitMQ/RabbitMQHandler.h | 9 +++ src/Storages/RabbitMQ/RabbitMQSettings.h | 4 +- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 86 +++++++++++------------ src/Storages/RabbitMQ/StorageRabbitMQ.h | 7 +- 4 files changed, 57 insertions(+), 49 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index 51cfbdc1144..d312a6c75c6 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -17,6 +17,15 @@ namespace Loop static const UInt8 STOP = 2; } + +class RabbitMQChannel : public AMQP::TcpChannel +{ +public: + RabbitMQChannel(AMQP::TcpConnection * connection) : TcpChannel(connection) {} + ~RabbitMQChannel() override { close(); } +}; + + class RabbitMQHandler : public AMQP::LibUvHandler { diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.h b/src/Storages/RabbitMQ/RabbitMQSettings.h index c6725903898..c44648ebd40 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.h +++ b/src/Storages/RabbitMQ/RabbitMQSettings.h @@ -20,11 +20,11 @@ namespace DB M(UInt64, rabbitmq_num_queues, 1, "The number of queues per consumer.", 0) \ M(String, rabbitmq_queue_base, "", "Base for queue names to be able to reopen non-empty queues in case of failure.", 0) \ M(String, rabbitmq_deadletter_exchange, "", "Exchange name to be passed as a dead-letter-exchange name.", 0) \ - M(Bool, rabbitmq_persistent, false, "If set, delivery mode will be set to 2 (makes messages 'persistent', durable).", 0) \ + M(Bool, rabbitmq_persistent, false, "For insert query messages will be made 'persistent', durable.", 0) \ M(UInt64, rabbitmq_skip_broken_messages, 0, "Skip at least this number of broken messages from RabbitMQ per block", 0) \ M(UInt64, rabbitmq_max_block_size, 0, "Number of row collected before flushing data from RabbitMQ.", 0) \ M(Milliseconds, rabbitmq_flush_interval_ms, 0, "Timeout for flushing data from RabbitMQ.", 0) \ - M(String, rabbitmq_vhost, "/", "RabbitMQ vhost.", 0) \ + M(String, rabbitmq_vhost, "/", "RabbitMQ vhost.", 0) #define LIST_OF_RABBITMQ_SETTINGS(M) \ RABBITMQ_RELATED_SETTINGS(M) \ diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 525a08784be..edfd16c55c2 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -259,46 +259,48 @@ size_t StorageRabbitMQ::getMaxBlockSize() const void StorageRabbitMQ::initRabbitMQ() { - setup_channel = std::make_shared(connection.get()); + RabbitMQChannel rabbit_channel(connection.get()); - initExchange(); - bindExchange(); + /// Main exchange -> Bridge exchange -> ( Sharding exchange ) -> Queues -> Consumers + + initExchange(rabbit_channel); + bindExchange(rabbit_channel); for (const auto i : ext::range(0, num_queues)) - bindQueue(i + 1); + bindQueue(i + 1, rabbit_channel); LOG_TRACE(log, "RabbitMQ setup completed"); - rabbit_is_ready = true; - setup_channel->close(); } -void StorageRabbitMQ::initExchange() +void StorageRabbitMQ::initExchange(RabbitMQChannel & rabbit_channel) { - /* Binding scheme is the following: client's exchange -> key bindings by routing key list -> bridge exchange (fanout) -> - * -> sharding exchange (only if needed) -> queues - */ - setup_channel->declareExchange(exchange_name, exchange_type, AMQP::durable) + /// Exchange hierarchy: + /// 1. Main exchange (defined with table settings - rabbitmq_exchange_name, rabbitmq_exchange_type). + /// 2. Bridge exchange (fanout). Used to easily disconnect main exchange and to simplify queue bindings. + /// 3. Sharding (or hash) exchange. Used in case of multiple queues. + /// 4. Consumer exchange. Just an alias for bridge_exchange or sharding exchange to know to what exchange + /// queues will be bound. + + /// All exchanges are declared with options: + /// 1. `durable` (survive RabbitMQ server restart) + /// 2. `autodelete` (auto delete in case of queue bindings are dropped). + + rabbit_channel.declareExchange(exchange_name, exchange_type, AMQP::durable) .onError([&](const char * message) { - /* This error can be a result of attempt to declare exchange if it was already declared but - * 1) with different exchange type. In this case can - * - manually delete previously declared exchange and create a new one. - * - throw an error that the exchange with this name but another type is already declared and ask client to delete it himself - * if it is not needed anymore or use another exchange name. - * 2) with different exchange settings. This can only happen if client himself declared exchange with the same name and - * specified its own settings, which differ from this implementation. - */ + /// This error can be a result of attempt to declare exchange if it was already declared but + /// 1) with different exchange type. + /// 2) with different exchange settings. throw Exception("Unable to declare exchange. Make sure specified exchange is not already declared. Error: " + std::string(message), ErrorCodes::CANNOT_DECLARE_RABBITMQ_EXCHANGE); }); - /// Bridge exchange is needed to easily disconnect consumer queues and also simplifies queue bindings - setup_channel->declareExchange(bridge_exchange, AMQP::fanout, AMQP::durable + AMQP::autodelete) + rabbit_channel.declareExchange(bridge_exchange, AMQP::fanout, AMQP::durable | AMQP::autodelete) .onError([&](const char * message) { - /// This error is not supposed to happen as this exchange name is always unique to type and its settings + /// This error is not supposed to happen as this exchange name is always unique to type and its settings. throw Exception( ErrorCodes::CANNOT_DECLARE_RABBITMQ_EXCHANGE, "Unable to declare bridge exchange ({}). Reason: {}", bridge_exchange, std::string(message)); }); @@ -309,26 +311,26 @@ void StorageRabbitMQ::initExchange() return; } - /* Change hash property because by default it will be routing key, which has to be an integer, but with support for any exchange - * type - routing keys might be of any type - */ AMQP::Table binding_arguments; + + /// Default routing key property in case of hash exchange is a routing key, which is required to be an integer. + /// Support for arbitrary exchange type (i.e. arbitary pattern of routing keys) requires to eliminate this dependency. + /// This settings changes hash propery to message_id. binding_arguments["hash-property"] = "message_id"; - /// Declare exchange for sharding. - setup_channel->declareExchange(sharding_exchange, AMQP::consistent_hash, AMQP::durable + AMQP::autodelete, binding_arguments) + /// Declare hash exchange for sharding. + rabbit_channel.declareExchange(sharding_exchange, AMQP::consistent_hash, AMQP::durable | AMQP::autodelete, binding_arguments) .onError([&](const char * message) { - /* This error can be a result of same reasons as above for exchange_name, i.e. it will mean that sharding exchange name appeared - * to be the same as some other exchange (which purpose is not for sharding). So probably actual error reason: queue_base parameter - * is bad. - */ + /// This error can be a result of same reasons as above for exchange_name, i.e. it will mean that sharding exchange name appeared + /// to be the same as some other exchange (which purpose is not for sharding). So probably actual error reason: queue_base parameter + /// is bad. throw Exception( ErrorCodes::CANNOT_DECLARE_RABBITMQ_EXCHANGE, "Unable to declare sharding exchange ({}). Reason: {}", sharding_exchange, std::string(message)); }); - setup_channel->bindExchange(bridge_exchange, sharding_exchange, routing_keys[0]) + rabbit_channel.bindExchange(bridge_exchange, sharding_exchange, routing_keys[0]) .onError([&](const char * message) { throw Exception( @@ -343,7 +345,7 @@ void StorageRabbitMQ::initExchange() } -void StorageRabbitMQ::bindExchange() +void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) { std::atomic binding_created = false; size_t bound_keys = 0; @@ -358,7 +360,7 @@ void StorageRabbitMQ::bindExchange() bind_headers[matching[0]] = matching[1]; } - setup_channel->bindExchange(exchange_name, bridge_exchange, routing_keys[0], bind_headers) + rabbit_channel.bindExchange(exchange_name, bridge_exchange, routing_keys[0], bind_headers) .onSuccess([&]() { binding_created = true; }) .onError([&](const char * message) { @@ -370,7 +372,7 @@ void StorageRabbitMQ::bindExchange() } else if (exchange_type == AMQP::ExchangeType::fanout || exchange_type == AMQP::ExchangeType::consistent_hash) { - setup_channel->bindExchange(exchange_name, bridge_exchange, routing_keys[0]) + rabbit_channel.bindExchange(exchange_name, bridge_exchange, routing_keys[0]) .onSuccess([&]() { binding_created = true; }) .onError([&](const char * message) { @@ -384,7 +386,7 @@ void StorageRabbitMQ::bindExchange() { for (const auto & routing_key : routing_keys) { - setup_channel->bindExchange(exchange_name, bridge_exchange, routing_key) + rabbit_channel.bindExchange(exchange_name, bridge_exchange, routing_key) .onSuccess([&]() { ++bound_keys; @@ -408,7 +410,7 @@ void StorageRabbitMQ::bindExchange() } -void StorageRabbitMQ::bindQueue(size_t queue_id) +void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channel) { std::atomic binding_created = false; @@ -424,7 +426,7 @@ void StorageRabbitMQ::bindQueue(size_t queue_id) * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for * fanout exchange it can be arbitrary */ - setup_channel->bindQueue(consumer_exchange, queue_name, std::to_string(queue_id)) + rabbit_channel.bindQueue(consumer_exchange, queue_name, std::to_string(queue_id)) .onSuccess([&] { binding_created = true; }) .onError([&](const char * message) { @@ -460,7 +462,7 @@ void StorageRabbitMQ::bindQueue(size_t queue_id) * specific queue when its name is specified in queue_base setting */ const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; - setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); + rabbit_channel.declareQueue(queue_name, AMQP::durable | AMQP::autodelete, queue_settings).onSuccess(success_callback).onError(error_callback); while (!binding_created) { @@ -537,8 +539,8 @@ void StorageRabbitMQ::unbindExchange() event_handler->updateLoopState(Loop::STOP); looping_task->deactivate(); - setup_channel = std::make_shared(connection.get()); - setup_channel->removeExchange(bridge_exchange) + RabbitMQChannel rabbit_channel(connection.get()); + rabbit_channel.removeExchange(bridge_exchange) .onSuccess([&]() { exchange_removed.store(true); @@ -552,8 +554,6 @@ void StorageRabbitMQ::unbindExchange() { event_handler->iterateLoop(); } - - setup_channel->close(); }); } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index eeda6b9fdca..524aac32fe5 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -113,7 +113,6 @@ private: std::atomic producer_id = 1; /// counter for producer buffer, needed for channel id std::atomic wait_confirm = true; /// needed to break waiting for confirmations for producer std::atomic exchange_removed = false, rabbit_is_ready = false; - ChannelPtr setup_channel; std::vector queues; std::once_flag flag; /// remove exchange only once @@ -141,9 +140,9 @@ private: void deactivateTask(BackgroundSchedulePool::TaskHolder & task, bool wait, bool stop_loop); void initRabbitMQ(); - void initExchange(); - void bindExchange(); - void bindQueue(size_t queue_id); + void initExchange(RabbitMQChannel & rabbit_channel); + void bindExchange(RabbitMQChannel & rabbit_channel); + void bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channel); bool restoreConnection(bool reconnecting); bool streamToViews(); From 13ba5287e3c37a45740052ed10f66f2657f8c080 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 4 May 2021 16:26:47 +0000 Subject: [PATCH 007/202] Add cleanup in case of drop table --- src/Storages/RabbitMQ/RabbitMQHandler.cpp | 12 +++++ src/Storages/RabbitMQ/RabbitMQHandler.h | 8 +++ .../RabbitMQ/ReadBufferFromRabbitMQConsumer.h | 1 + src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 50 +++++++++++++++++-- src/Storages/RabbitMQ/StorageRabbitMQ.h | 10 ++++ .../integration/test_storage_rabbitmq/test.py | 38 ++++++++++++++ 6 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.cpp b/src/Storages/RabbitMQ/RabbitMQHandler.cpp index d08b4806db8..9b5beb0c3f2 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.cpp +++ b/src/Storages/RabbitMQ/RabbitMQHandler.cpp @@ -53,4 +53,16 @@ void RabbitMQHandler::iterateLoop() uv_run(loop, UV_RUN_NOWAIT); } +int RabbitMQHandler::startBlockingLoop() +{ + /// Return non-zero value, if uv_stop was called while there are still active events. + int ret = uv_run(loop, UV_RUN_DEFAULT); + return ret; +} + +void RabbitMQHandler::stopLoop() +{ + uv_stop(loop); +} + } diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index d312a6c75c6..27027e7ab42 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -35,9 +35,17 @@ public: void onError(AMQP::TcpConnection * connection, const char * message) override; void onReady(AMQP::TcpConnection * connection) override; + /// Loop for background thread worker. void startLoop(); + + /// Loop to wait for small tasks in a non-blocking mode. void iterateLoop(); + /// Loop to wait for small tasks in a blocking mode. + int startBlockingLoop(); + + void stopLoop(); + bool connectionRunning() { return connection_running.load(); } bool loopRunning() { return loop_running.load(); } diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 23e1d3f0649..e4c76470ed1 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -56,6 +56,7 @@ public: ChannelPtr & getChannel() { return consumer_channel; } void setupChannel(); bool needChannelUpdate(); + void closeChannel() { consumer_channel->close(); } void updateQueues(std::vector & queues_) { queues = queues_; } size_t queuesCount() { return queues.size(); } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index edfd16c55c2..842a569e633 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -462,7 +462,9 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channe * specific queue when its name is specified in queue_base setting */ const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; - rabbit_channel.declareQueue(queue_name, AMQP::durable | AMQP::autodelete, queue_settings).onSuccess(success_callback).onError(error_callback); + + /// AMQP::autodelete setting is not allowd, because in case of server restart there will be no consumers and deleting queues should not take place. + rabbit_channel.declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); while (!binding_created) { @@ -644,12 +646,27 @@ void StorageRabbitMQ::shutdown() stream_cancelled = true; wait_confirm = false; - deactivateTask(streaming_task, true, false); - deactivateTask(looping_task, true, true); + /// In case it has not yet been able to setup connection; deactivateTask(connection_task, true, false); + /// The order of deactivating tasks is important: wait for streamingToViews() func to finish and + /// then wait for background event loop to finish. + deactivateTask(streaming_task, true, false); + deactivateTask(looping_task, true, true); + + if (drop_table) + { + for (auto & buffer : buffers) + buffer->closeChannel(); + cleanupRabbitMQ(); + } + + /// It is important to close connection here - before removing consumer buffers, because + /// it will finish and clean callbacks, which might use those buffers data. connection->close(); + /// Connection is not closed immediately - it requires the loop to shutdown it properly and to + /// finish all callbacks. size_t cnt_retries = 0; while (!connection->closed() && cnt_retries++ != RETRIES_MAX) event_handler->iterateLoop(); @@ -663,6 +680,33 @@ void StorageRabbitMQ::shutdown() } +/// The only thing publishers are supposed to be aware of is _exchanges_ and queues are a responsibility of a consumer. +/// Therefore, if a table is droppped, a clean up is needed. +void StorageRabbitMQ::cleanupRabbitMQ() const +{ + RabbitMQChannel rabbit_channel(connection.get()); + for (const auto & queue : queues) + { + /// AMQP::ifunused is needed, because it is possible to share queues between multiple tables and dropping + /// on of them should not affect others. + /// AMQP::ifempty is not used on purpose. + + rabbit_channel.removeQueue(queue, AMQP::ifunused) + .onSuccess([&](uint32_t num_messages) + { + LOG_TRACE(log, "Successfully deleted queue {}, messages contained {}", queue, num_messages); + event_handler->stopLoop(); + }) + .onError([&](const char * message) + { + LOG_ERROR(log, "Failed to delete queue {}. Error message: {}", queue, message); + event_handler->stopLoop(); + }); + } + event_handler->startBlockingLoop(); +} + + void StorageRabbitMQ::pushReadBuffer(ConsumerBufferPtr buffer) { std::lock_guard lock(buffers_mutex); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 524aac32fe5..6df8bd95276 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -33,6 +33,13 @@ public: void startup() override; void shutdown() override; + /// This is a bad way to let storage know in shutdown() that table is going to be dropped. There are some actions which need + /// to be done only when table is dropped (not when detached). Also connection must be closed only in shutdown, but those + /// actions require an open connection. Therefore there needs to be a way inside shutdown() method to know whether it is called + /// because of drop query. And drop() method is not suitable at all, because it will not only require to reopen connection, but also + /// it can be called considerable time after table is dropped (for example, in case of Atomic database), which is not appropriate for the case. + void checkTableCanBeDropped() const override { drop_table = true; } + /// Always return virtual columns in addition to required columns Pipe read( const Names & column_names, @@ -123,6 +130,7 @@ private: std::atomic stream_cancelled{false}; size_t read_attempts = 0; + mutable bool drop_table = false; ConsumerBufferPtr createReadBuffer(); @@ -140,6 +148,8 @@ private: void deactivateTask(BackgroundSchedulePool::TaskHolder & task, bool wait, bool stop_loop); void initRabbitMQ(); + void cleanupRabbitMQ() const; + void initExchange(RabbitMQChannel & rabbit_channel); void bindExchange(RabbitMQChannel & rabbit_channel); void bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channel); diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index cab7685d96c..3e7096a4c55 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1994,6 +1994,44 @@ def test_rabbitmq_vhost(rabbitmq_cluster): break +@pytest.mark.timeout(120) +def test_rabbitmq_drop_table_properly(rabbitmq_cluster): + instance.query('CREATE DATABASE test_database') + instance.query(''' + CREATE TABLE test_database.rabbitmq_drop (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = 'drop', + rabbitmq_format = 'JSONEachRow', + rabbitmq_queue_base = 'rabbit_queue' + ''') + + credentials = pika.PlainCredentials('root', 'clickhouse') + parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + + channel.basic_publish(exchange='drop', routing_key='', body=json.dumps({'key': 1, 'value': 2})) + while True: + result = instance.query('SELECT * FROM test_database.rabbitmq_drop ORDER BY key', ignore_error=True) + if result == "1\t2\n": + break + + exists = channel.queue_declare(queue='rabbit_queue', passive=True) + assert(exists) + + instance.query("DROP TABLE test_database.rabbitmq_drop") + time.sleep(30) + instance.query("DROP DATABASE test_database") + + try: + exists = channel.queue_declare(callback, queue='rabbit_queue', passive=True) + except Exception as e: + exists = False + + assert(not exists) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From a3feaa48f4c9431c0bd5d5709d52084e1aa1562e Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 4 May 2021 18:57:49 +0000 Subject: [PATCH 008/202] Allow user to define specific queue settings --- src/Storages/RabbitMQ/RabbitMQSettings.h | 3 +- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 62 ++++++++++++----- src/Storages/RabbitMQ/StorageRabbitMQ.h | 3 +- .../integration/test_storage_rabbitmq/test.py | 66 +++++++++++++------ 4 files changed, 98 insertions(+), 36 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.h b/src/Storages/RabbitMQ/RabbitMQSettings.h index c44648ebd40..16d47bda81e 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.h +++ b/src/Storages/RabbitMQ/RabbitMQSettings.h @@ -24,7 +24,8 @@ namespace DB M(UInt64, rabbitmq_skip_broken_messages, 0, "Skip at least this number of broken messages from RabbitMQ per block", 0) \ M(UInt64, rabbitmq_max_block_size, 0, "Number of row collected before flushing data from RabbitMQ.", 0) \ M(Milliseconds, rabbitmq_flush_interval_ms, 0, "Timeout for flushing data from RabbitMQ.", 0) \ - M(String, rabbitmq_vhost, "/", "RabbitMQ vhost.", 0) + M(String, rabbitmq_vhost, "/", "RabbitMQ vhost.", 0) \ + M(String, rabbitmq_queue_settings_list, "", "A list of rabbitmq queue settings", 0) \ #define LIST_OF_RABBITMQ_SETTINGS(M) \ RABBITMQ_RELATED_SETTINGS(M) \ diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 842a569e633..2aab01bed88 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -79,12 +79,13 @@ StorageRabbitMQ::StorageRabbitMQ( , exchange_name(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_exchange_name.value)) , format_name(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_format.value)) , exchange_type(defineExchangeType(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_exchange_type.value))) - , routing_keys(parseRoutingKeys(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_routing_key_list.value))) + , routing_keys(parseSettings(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_routing_key_list.value))) , row_delimiter(rabbitmq_settings->rabbitmq_row_delimiter.value) , schema_name(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_schema.value)) , num_consumers(rabbitmq_settings->rabbitmq_num_consumers.value) , num_queues(rabbitmq_settings->rabbitmq_num_queues.value) , queue_base(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_queue_base.value)) + , queue_settings_list(parseSettings(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_queue_settings_list.value))) , deadletter_exchange(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_deadletter_exchange.value)) , persistent(rabbitmq_settings->rabbitmq_persistent.value) , hash_exchange(num_consumers > 1 || num_queues > 1) @@ -147,10 +148,12 @@ StorageRabbitMQ::StorageRabbitMQ( } -Names StorageRabbitMQ::parseRoutingKeys(String routing_key_list) +Names StorageRabbitMQ::parseSettings(String settings_list) { Names result; - boost::split(result, routing_key_list, [](char c){ return c == ','; }); + if (settings_list.empty()) + return result; + boost::split(result, settings_list, [](char c){ return c == ','; }); for (String & key : result) boost::trim(key); @@ -250,11 +253,11 @@ void StorageRabbitMQ::deactivateTask(BackgroundSchedulePool::TaskHolder & task, size_t StorageRabbitMQ::getMaxBlockSize() const - { +{ return rabbitmq_settings->rabbitmq_max_block_size.changed ? rabbitmq_settings->rabbitmq_max_block_size.value : (getContext()->getSettingsRef().max_insert_block_size.value / num_consumers); - } +} void StorageRabbitMQ::initRabbitMQ() @@ -451,19 +454,50 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channe AMQP::Table queue_settings; - queue_settings["x-max-length"] = queue_size; + /// Check user-defined settings. + if (!queue_settings_list.empty()) + { + for (const auto & setting : queue_settings_list) + { + Strings setting_values; + splitInto<'='>(setting_values, setting); + assert(setting_values.size() == 2); + String key = setting_values[0], value = setting_values[1]; - if (!deadletter_exchange.empty()) + std::unordered_set integer_settings = {"x-max-length", "x-max-length-bytes", "x-message-ttl", "x-expires", "x-priority", "x-max-priority"}; + std::unordered_set string_settings = {"x-overflow", "x-dead-letter-exchange", "x-queue-type"}; + + if (integer_settings.find(key) != integer_settings.end()) + queue_settings[key] = parse(value); + else if (string_settings.find(key) != string_settings.end()) + queue_settings[key] = value; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported queue setting: {}", value); + } + } + + /// Impose default settings if there are no user-defined settings. + if (!queue_settings.contains("x-max-length")) + { + queue_settings["x-max-length"] = queue_size; + } + if (!queue_settings.contains("x-dead-letter-exchange") && !deadletter_exchange.empty()) + { queue_settings["x-dead-letter-exchange"] = deadletter_exchange; - else + } + else if (!queue_settings.contains("x-overflow")) + { + /// Define x-overflow only if there is not x-dead-letter-exchange, because it will overwrite the expected behaviour. queue_settings["x-overflow"] = "reject-publish"; + } - /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one - * specific queue when its name is specified in queue_base setting - */ + /// If queue_base - a single name, then it can be used as one specific queue, from which to read. + /// Otherwise it is used as a generator (unique for current table) of queue names, because it allows to + /// maximize performance - via setting `rabbitmq_num_queues`. const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; - /// AMQP::autodelete setting is not allowd, because in case of server restart there will be no consumers and deleting queues should not take place. + /// AMQP::autodelete setting is not allowd, because in case of server restart there will be no consumers + /// and deleting queues should not take place. rabbit_channel.declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); while (!binding_created) @@ -1017,7 +1051,6 @@ void registerStorageRabbitMQ(StorageFactory & factory) CHECK_RABBITMQ_STORAGE_ARGUMENT(1, rabbitmq_host_port) CHECK_RABBITMQ_STORAGE_ARGUMENT(2, rabbitmq_exchange_name) CHECK_RABBITMQ_STORAGE_ARGUMENT(3, rabbitmq_format) - CHECK_RABBITMQ_STORAGE_ARGUMENT(4, rabbitmq_exchange_type) CHECK_RABBITMQ_STORAGE_ARGUMENT(5, rabbitmq_routing_key_list) CHECK_RABBITMQ_STORAGE_ARGUMENT(6, rabbitmq_row_delimiter) @@ -1027,12 +1060,11 @@ void registerStorageRabbitMQ(StorageFactory & factory) CHECK_RABBITMQ_STORAGE_ARGUMENT(10, rabbitmq_queue_base) CHECK_RABBITMQ_STORAGE_ARGUMENT(11, rabbitmq_deadletter_exchange) CHECK_RABBITMQ_STORAGE_ARGUMENT(12, rabbitmq_persistent) - CHECK_RABBITMQ_STORAGE_ARGUMENT(13, rabbitmq_skip_broken_messages) CHECK_RABBITMQ_STORAGE_ARGUMENT(14, rabbitmq_max_block_size) CHECK_RABBITMQ_STORAGE_ARGUMENT(15, rabbitmq_flush_interval_ms) - CHECK_RABBITMQ_STORAGE_ARGUMENT(16, rabbitmq_vhost) + CHECK_RABBITMQ_STORAGE_ARGUMENT(16, rabbitmq_queue_settings_list) #undef CHECK_RABBITMQ_STORAGE_ARGUMENT diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 6df8bd95276..fcf300b30e5 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -91,6 +91,7 @@ private: size_t num_consumers; size_t num_queues; String queue_base; + Names queue_settings_list; const String deadletter_exchange; const bool persistent; @@ -139,7 +140,7 @@ private: void loopingFunc(); void connectionFunc(); - static Names parseRoutingKeys(String routing_key_list); + static Names parseSettings(String routing_key_list); static AMQP::ExchangeType defineExchangeType(String exchange_type_); static String getTableBasedName(String name, const StorageID & table_id); diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 3e7096a4c55..eb3e21bdccb 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -26,24 +26,13 @@ rabbitmq_id = '' # Helpers def check_rabbitmq_is_available(): - p = subprocess.Popen(('docker', - 'exec', - '-i', - rabbitmq_id, - 'rabbitmqctl', - 'await_startup'), - stdout=subprocess.PIPE) + p = subprocess.Popen(('docker', 'exec', '-i', rabbitmq_id, 'rabbitmqctl', 'await_startup'), stdout=subprocess.PIPE) p.communicate() return p.returncode == 0 def enable_consistent_hash_plugin(): - p = subprocess.Popen(('docker', - 'exec', - '-i', - rabbitmq_id, - "rabbitmq-plugins", "enable", "rabbitmq_consistent_hash_exchange"), - stdout=subprocess.PIPE) + p = subprocess.Popen(('docker', 'exec', '-i', rabbitmq_id, "rabbitmq-plugins", "enable", "rabbitmq_consistent_hash_exchange"), stdout=subprocess.PIPE) p.communicate() return p.returncode == 0 @@ -1835,7 +1824,7 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): cancel.set() instance.query(''' - DROP TABLE test.rabbitmq; + DETACH TABLE test.rabbitmq; ''') while int(instance.query("SELECT count() FROM system.tables WHERE database='test' AND name='rabbitmq'")) == 1: @@ -1996,9 +1985,8 @@ def test_rabbitmq_vhost(rabbitmq_cluster): @pytest.mark.timeout(120) def test_rabbitmq_drop_table_properly(rabbitmq_cluster): - instance.query('CREATE DATABASE test_database') instance.query(''' - CREATE TABLE test_database.rabbitmq_drop (key UInt64, value UInt64) + CREATE TABLE test.rabbitmq_drop (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'drop', @@ -2013,16 +2001,15 @@ def test_rabbitmq_drop_table_properly(rabbitmq_cluster): channel.basic_publish(exchange='drop', routing_key='', body=json.dumps({'key': 1, 'value': 2})) while True: - result = instance.query('SELECT * FROM test_database.rabbitmq_drop ORDER BY key', ignore_error=True) + result = instance.query('SELECT * FROM test.rabbitmq_drop ORDER BY key', ignore_error=True) if result == "1\t2\n": break exists = channel.queue_declare(queue='rabbit_queue', passive=True) assert(exists) - instance.query("DROP TABLE test_database.rabbitmq_drop") + instance.query("DROP TABLE test.rabbitmq_drop") time.sleep(30) - instance.query("DROP DATABASE test_database") try: exists = channel.queue_declare(callback, queue='rabbit_queue', passive=True) @@ -2032,6 +2019,47 @@ def test_rabbitmq_drop_table_properly(rabbitmq_cluster): assert(not exists) +@pytest.mark.timeout(120) +def test_rabbitmq_queue_settings(rabbitmq_cluster): + instance.query(''' + CREATE TABLE test.rabbitmq_settings (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = 'rabbit_exchange', + rabbitmq_format = 'JSONEachRow', + rabbitmq_queue_base = 'rabbit_queue', + rabbitmq_queue_settings_list = 'x-max-length=10,x-overflow=reject-publish' + ''') + + credentials = pika.PlainCredentials('root', 'clickhouse') + parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + + for i in range(50): + channel.basic_publish(exchange='rabbit_exchange', routing_key='', body=json.dumps({'key': 1, 'value': 2})) + connection.close() + + instance.query(''' + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.rabbitmq_settings; + ''') + + time.sleep(5) + + result = instance.query('SELECT count() FROM test.rabbitmq_settings', ignore_error=True) + while int(result) != 10: + time.sleep(0.5) + result = instance.query('SELECT count() FROM test.view', ignore_error=True) + + # queue size is 10, but 50 messages were sent, they will be dropped (setting x-overflow = reject-publish) and only 10 will remain. + assert(int(result) == 10) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 2112521304e33f2f343306f3ee59e794be520eb6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 4 May 2021 19:54:16 +0000 Subject: [PATCH 009/202] Allow all RabbitMQ setup to be done only by user --- src/Storages/RabbitMQ/RabbitMQSettings.h | 1 + src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 11 ++- src/Storages/RabbitMQ/StorageRabbitMQ.h | 8 +++ .../integration/test_storage_rabbitmq/test.py | 71 +++++++++++++++---- 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.h b/src/Storages/RabbitMQ/RabbitMQSettings.h index 16d47bda81e..185ee2a6eb2 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.h +++ b/src/Storages/RabbitMQ/RabbitMQSettings.h @@ -26,6 +26,7 @@ namespace DB M(Milliseconds, rabbitmq_flush_interval_ms, 0, "Timeout for flushing data from RabbitMQ.", 0) \ M(String, rabbitmq_vhost, "/", "RabbitMQ vhost.", 0) \ M(String, rabbitmq_queue_settings_list, "", "A list of rabbitmq queue settings", 0) \ + M(Bool, rabbitmq_queue_consume, false, "Use user-defined queues and do not make any RabbitMQ setup: declaring exchanges, queues, bindings", 0) \ #define LIST_OF_RABBITMQ_SETTINGS(M) \ RABBITMQ_RELATED_SETTINGS(M) \ diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 2aab01bed88..c1e164c9868 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -88,6 +88,7 @@ StorageRabbitMQ::StorageRabbitMQ( , queue_settings_list(parseSettings(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_queue_settings_list.value))) , deadletter_exchange(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_deadletter_exchange.value)) , persistent(rabbitmq_settings->rabbitmq_persistent.value) + , use_user_setup(rabbitmq_settings->rabbitmq_queue_consume.value) , hash_exchange(num_consumers > 1 || num_queues > 1) , log(&Poco::Logger::get("StorageRabbitMQ (" + table_id_.table_name + ")")) , address(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_host_port.value)) @@ -262,6 +263,13 @@ size_t StorageRabbitMQ::getMaxBlockSize() const void StorageRabbitMQ::initRabbitMQ() { + if (use_user_setup) + { + queues.emplace_back(queue_base); + rabbit_is_ready = true; + return; + } + RabbitMQChannel rabbit_channel(connection.get()); /// Main exchange -> Bridge exchange -> ( Sharding exchange ) -> Queues -> Consumers @@ -1064,7 +1072,8 @@ void registerStorageRabbitMQ(StorageFactory & factory) CHECK_RABBITMQ_STORAGE_ARGUMENT(14, rabbitmq_max_block_size) CHECK_RABBITMQ_STORAGE_ARGUMENT(15, rabbitmq_flush_interval_ms) CHECK_RABBITMQ_STORAGE_ARGUMENT(16, rabbitmq_vhost) - CHECK_RABBITMQ_STORAGE_ARGUMENT(16, rabbitmq_queue_settings_list) + CHECK_RABBITMQ_STORAGE_ARGUMENT(17, rabbitmq_queue_settings_list) + CHECK_RABBITMQ_STORAGE_ARGUMENT(18, rabbitmq_queue_consume) #undef CHECK_RABBITMQ_STORAGE_ARGUMENT diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index fcf300b30e5..664959668b3 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -93,8 +93,15 @@ private: String queue_base; Names queue_settings_list; const String deadletter_exchange; + + /// For insert query. Mark messages as durable. const bool persistent; + /// A table setting. It is possible not to perform any RabbitMQ setup, which is supposed to be consumer-side setup: + /// declaring exchanges, queues, bindings. Instead everything needed from RabbitMQ table is to connect to a specific queue. + /// This solution disables all optimizations and is not really optimal, but allows user to fully control all RabbitMQ setup. + bool use_user_setup; + bool hash_exchange; Poco::Logger * log; String address; @@ -116,6 +123,7 @@ private: /// maximum number of messages in RabbitMQ queue (x-max-length). Also used /// to setup size of inner buffer for received messages uint32_t queue_size; + String sharding_exchange, bridge_exchange, consumer_exchange; size_t consumer_id = 0; /// counter for consumer buffer, needed for channel id std::atomic producer_id = 1; /// counter for producer buffer, needed for channel id diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index eb3e21bdccb..41e6421691d 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1823,23 +1823,12 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster): cancel.set() - instance.query(''' - DETACH TABLE test.rabbitmq; - ''') + instance.query('DETACH TABLE test.rabbitmq;') while int(instance.query("SELECT count() FROM system.tables WHERE database='test' AND name='rabbitmq'")) == 1: time.sleep(1) - instance.query(''' - CREATE TABLE test.rabbitmq (key UInt64, value UInt64) - ENGINE = RabbitMQ - SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', - rabbitmq_exchange_name = 'block', - rabbitmq_format = 'JSONEachRow', - rabbitmq_max_block_size = 100, - rabbitmq_queue_base = 'block', - rabbitmq_row_delimiter = '\\n'; - ''') + instance.query('ATTACH TABLE test.rabbitmq;') while int(instance.query('SELECT uniqExact(key) FROM test.view')) < i[0]: time.sleep(1) @@ -2060,6 +2049,62 @@ def test_rabbitmq_queue_settings(rabbitmq_cluster): assert(int(result) == 10) +@pytest.mark.timeout(120) +def test_rabbitmq_queue_consume(rabbitmq_cluster): + credentials = pika.PlainCredentials('root', 'clickhouse') + parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + channel.queue_declare(queue='rabbit_queue', durable=True) + #channel.basic_publish(exchange='', routing_key='rabbit_queue', body=json.dumps({'key': 1, 'value': 2})) + + i = [0] + messages_num = 1000 + def produce(): + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + messages = [] + for _ in range(messages_num): + message = json.dumps({'key': i[0], 'value': i[0]}) + channel.basic_publish(exchange='', routing_key='rabbit_queue', body=message) + i[0] += 1 + + threads = [] + threads_num = 10 + for _ in range(threads_num): + threads.append(threading.Thread(target=produce)) + for thread in threads: + time.sleep(random.uniform(0, 1)) + thread.start() + + instance.query(''' + CREATE TABLE test.rabbitmq_queue (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = 'drop', + rabbitmq_format = 'JSONEachRow', + rabbitmq_queue_base = 'rabbit_queue', + rabbitmq_queue_consume = 1; + + DROP TABLE IF EXISTS test.view; + DROP TABLE IF EXISTS test.consumer; + CREATE TABLE test.view (key UInt64, value UInt64) + ENGINE = MergeTree ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer TO test.view AS + SELECT * FROM test.rabbitmq_queue; + ''') + + result = '' + while True: + result = instance.query('SELECT count() FROM test.view') + if int(result) == messages_num * threads_num: + break + time.sleep(1) + + for thread in threads: + thread.join() + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 973ee4e9c4b693309e88299fcc5d439b38c869cb Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 4 May 2021 19:57:45 +0000 Subject: [PATCH 010/202] Remove strange code, some fixes --- src/Storages/RabbitMQ/RabbitMQHandler.cpp | 2 + src/Storages/RabbitMQ/RabbitMQHandler.h | 2 + src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 46 +++++++++---------- .../integration/test_storage_rabbitmq/test.py | 5 +- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.cpp b/src/Storages/RabbitMQ/RabbitMQHandler.cpp index 9b5beb0c3f2..07348d9c068 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.cpp +++ b/src/Storages/RabbitMQ/RabbitMQHandler.cpp @@ -53,6 +53,8 @@ void RabbitMQHandler::iterateLoop() uv_run(loop, UV_RUN_NOWAIT); } +/// Do not need synchronization as in iterateLoop(), because this method is used only for +/// initial RabbitMQ setup - at this point there is no background loop thread. int RabbitMQHandler::startBlockingLoop() { /// Return non-zero value, if uv_stop was called while there are still active events. diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index 27027e7ab42..bd7b4526e87 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -39,9 +39,11 @@ public: void startLoop(); /// Loop to wait for small tasks in a non-blocking mode. + /// Adds synchronization with main background loop. void iterateLoop(); /// Loop to wait for small tasks in a blocking mode. + /// No synchronization is done with the main loop thread. int startBlockingLoop(); void stopLoop(); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index c1e164c9868..a12569ddf40 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -76,22 +76,22 @@ StorageRabbitMQ::StorageRabbitMQ( : IStorage(table_id_) , WithContext(context_->getGlobalContext()) , rabbitmq_settings(std::move(rabbitmq_settings_)) - , exchange_name(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_exchange_name.value)) - , format_name(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_format.value)) - , exchange_type(defineExchangeType(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_exchange_type.value))) - , routing_keys(parseSettings(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_routing_key_list.value))) + , exchange_name(rabbitmq_settings->rabbitmq_exchange_name.value) + , format_name(rabbitmq_settings->rabbitmq_format.value) + , exchange_type(defineExchangeType(rabbitmq_settings->rabbitmq_exchange_type.value)) + , routing_keys(parseSettings(rabbitmq_settings->rabbitmq_routing_key_list.value)) , row_delimiter(rabbitmq_settings->rabbitmq_row_delimiter.value) - , schema_name(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_schema.value)) + , schema_name(rabbitmq_settings->rabbitmq_schema.value) , num_consumers(rabbitmq_settings->rabbitmq_num_consumers.value) , num_queues(rabbitmq_settings->rabbitmq_num_queues.value) - , queue_base(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_queue_base.value)) - , queue_settings_list(parseSettings(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_queue_settings_list.value))) - , deadletter_exchange(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_deadletter_exchange.value)) + , queue_base(rabbitmq_settings->rabbitmq_queue_base.value) + , queue_settings_list(parseSettings(rabbitmq_settings->rabbitmq_queue_settings_list.value)) + , deadletter_exchange(rabbitmq_settings->rabbitmq_deadletter_exchange.value) , persistent(rabbitmq_settings->rabbitmq_persistent.value) , use_user_setup(rabbitmq_settings->rabbitmq_queue_consume.value) , hash_exchange(num_consumers > 1 || num_queues > 1) , log(&Poco::Logger::get("StorageRabbitMQ (" + table_id_.table_name + ")")) - , address(getContext()->getMacros()->expand(rabbitmq_settings->rabbitmq_host_port.value)) + , address(rabbitmq_settings->rabbitmq_host_port.value) , parsed_address(parseAddress(address, 5672)) , login_password(std::make_pair( getContext()->getConfigRef().getString("rabbitmq.username"), @@ -358,7 +358,6 @@ void StorageRabbitMQ::initExchange(RabbitMQChannel & rabbit_channel) void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) { - std::atomic binding_created = false; size_t bound_keys = 0; if (exchange_type == AMQP::ExchangeType::headers) @@ -372,7 +371,7 @@ void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) } rabbit_channel.bindExchange(exchange_name, bridge_exchange, routing_keys[0], bind_headers) - .onSuccess([&]() { binding_created = true; }) + .onSuccess([&]() { event_handler->stopLoop(); }) .onError([&](const char * message) { throw Exception( @@ -384,7 +383,7 @@ void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) else if (exchange_type == AMQP::ExchangeType::fanout || exchange_type == AMQP::ExchangeType::consistent_hash) { rabbit_channel.bindExchange(exchange_name, bridge_exchange, routing_keys[0]) - .onSuccess([&]() { binding_created = true; }) + .onSuccess([&]() { event_handler->stopLoop(); }) .onError([&](const char * message) { throw Exception( @@ -402,7 +401,7 @@ void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) { ++bound_keys; if (bound_keys == routing_keys.size()) - binding_created = true; + event_handler->stopLoop(); }) .onError([&](const char * message) { @@ -414,17 +413,12 @@ void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) } } - while (!binding_created) - { - event_handler->iterateLoop(); - } + event_handler->startBlockingLoop(); } void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channel) { - std::atomic binding_created = false; - auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) { queues.emplace_back(queue_name); @@ -438,7 +432,7 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channe * fanout exchange it can be arbitrary */ rabbit_channel.bindQueue(consumer_exchange, queue_name, std::to_string(queue_id)) - .onSuccess([&] { binding_created = true; }) + .onSuccess([&] { event_handler->stopLoop(); }) .onError([&](const char * message) { throw Exception( @@ -507,11 +501,7 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channe /// AMQP::autodelete setting is not allowd, because in case of server restart there will be no consumers /// and deleting queues should not take place. rabbit_channel.declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); - - while (!binding_created) - { - event_handler->iterateLoop(); - } + event_handler->startBlockingLoop(); } @@ -726,6 +716,9 @@ void StorageRabbitMQ::shutdown() /// Therefore, if a table is droppped, a clean up is needed. void StorageRabbitMQ::cleanupRabbitMQ() const { + if (use_user_setup) + return; + RabbitMQChannel rabbit_channel(connection.get()); for (const auto & queue : queues) { @@ -746,6 +739,9 @@ void StorageRabbitMQ::cleanupRabbitMQ() const }); } event_handler->startBlockingLoop(); + + /// Also there is no need to cleanup exchanges as they were created with AMQP::autodelete option. Once queues + /// are removed, exchanges will also be cleaned. } diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 41e6421691d..f6274db3d4c 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2045,6 +2045,8 @@ def test_rabbitmq_queue_settings(rabbitmq_cluster): time.sleep(0.5) result = instance.query('SELECT count() FROM test.view', ignore_error=True) + instance.query('DROP TABLE test.rabbitmq_settings') + # queue size is 10, but 50 messages were sent, they will be dropped (setting x-overflow = reject-publish) and only 10 will remain. assert(int(result) == 10) @@ -2056,7 +2058,6 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): connection = pika.BlockingConnection(parameters) channel = connection.channel() channel.queue_declare(queue='rabbit_queue', durable=True) - #channel.basic_publish(exchange='', routing_key='rabbit_queue', body=json.dumps({'key': 1, 'value': 2})) i = [0] messages_num = 1000 @@ -2104,6 +2105,8 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): for thread in threads: thread.join() + instance.query('DROP TABLE test.rabbitmq_queue') + if __name__ == '__main__': cluster.start() From 0b6fb7660b3ea76044d2780ba8e85b845249e30f Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 5 May 2021 07:52:21 +0000 Subject: [PATCH 011/202] Fix checks --- src/Storages/RabbitMQ/RabbitMQHandler.cpp | 6 ++---- src/Storages/RabbitMQ/RabbitMQHandler.h | 2 +- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 18 +++++++++--------- src/Storages/RabbitMQ/StorageRabbitMQ.h | 2 +- .../integration/test_storage_rabbitmq/test.py | 1 - 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.cpp b/src/Storages/RabbitMQ/RabbitMQHandler.cpp index 07348d9c068..c994ab22494 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.cpp +++ b/src/Storages/RabbitMQ/RabbitMQHandler.cpp @@ -55,11 +55,9 @@ void RabbitMQHandler::iterateLoop() /// Do not need synchronization as in iterateLoop(), because this method is used only for /// initial RabbitMQ setup - at this point there is no background loop thread. -int RabbitMQHandler::startBlockingLoop() +void RabbitMQHandler::startBlockingLoop() { - /// Return non-zero value, if uv_stop was called while there are still active events. - int ret = uv_run(loop, UV_RUN_DEFAULT); - return ret; + uv_run(loop, UV_RUN_DEFAULT); } void RabbitMQHandler::stopLoop() diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index bd7b4526e87..f1c7afffc5a 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -44,7 +44,7 @@ public: /// Loop to wait for small tasks in a blocking mode. /// No synchronization is done with the main loop thread. - int startBlockingLoop(); + void startBlockingLoop(); void stopLoop(); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index a12569ddf40..130274e2161 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -195,7 +195,7 @@ String StorageRabbitMQ::getTableBasedName(String name, const StorageID & table_i std::shared_ptr StorageRabbitMQ::addSettings(ContextPtr local_context) const { auto modified_context = Context::createCopy(local_context); - modified_context->setSetting("input_format_skip_unknown_fields", true); + modified_context->setSetting("input_format_skip_unknown_fields", 1); modified_context->setSetting("input_format_allow_errors_ratio", 0.); modified_context->setSetting("input_format_allow_errors_num", rabbitmq_settings->rabbitmq_skip_broken_messages.value); @@ -325,8 +325,8 @@ void StorageRabbitMQ::initExchange(RabbitMQChannel & rabbit_channel) AMQP::Table binding_arguments; /// Default routing key property in case of hash exchange is a routing key, which is required to be an integer. - /// Support for arbitrary exchange type (i.e. arbitary pattern of routing keys) requires to eliminate this dependency. - /// This settings changes hash propery to message_id. + /// Support for arbitrary exchange type (i.e. arbitrary pattern of routing keys) requires to eliminate this dependency. + /// This settings changes hash property to message_id. binding_arguments["hash-property"] = "message_id"; /// Declare hash exchange for sharding. @@ -470,7 +470,7 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channe std::unordered_set string_settings = {"x-overflow", "x-dead-letter-exchange", "x-queue-type"}; if (integer_settings.find(key) != integer_settings.end()) - queue_settings[key] = parse(value); + queue_settings[key] = parse(value); else if (string_settings.find(key) != string_settings.end()) queue_settings[key] = value; else @@ -498,7 +498,7 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channe /// maximize performance - via setting `rabbitmq_num_queues`. const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; - /// AMQP::autodelete setting is not allowd, because in case of server restart there will be no consumers + /// AMQP::autodelete setting is not allowed, because in case of server restart there will be no consumers /// and deleting queues should not take place. rabbit_channel.declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); event_handler->startBlockingLoop(); @@ -713,7 +713,7 @@ void StorageRabbitMQ::shutdown() /// The only thing publishers are supposed to be aware of is _exchanges_ and queues are a responsibility of a consumer. -/// Therefore, if a table is droppped, a clean up is needed. +/// Therefore, if a table is dropped, a clean up is needed. void StorageRabbitMQ::cleanupRabbitMQ() const { if (use_user_setup) @@ -1038,7 +1038,7 @@ void registerStorageRabbitMQ(StorageFactory & factory) // Check arguments and settings #define CHECK_RABBITMQ_STORAGE_ARGUMENT(ARG_NUM, ARG_NAME) \ /* One of the three required arguments is not specified */ \ - if (args_count < (ARG_NUM) && (ARG_NUM) <= 3 && !rabbitmq_settings->ARG_NAME.changed) \ + if (args_count < (ARG_NUM) && (ARG_NUM) <= 2 && !rabbitmq_settings->ARG_NAME.changed) \ { \ throw Exception("Required parameter '" #ARG_NAME "' for storage RabbitMQ not specified", \ ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); \ @@ -1053,8 +1053,8 @@ void registerStorageRabbitMQ(StorageFactory & factory) } CHECK_RABBITMQ_STORAGE_ARGUMENT(1, rabbitmq_host_port) - CHECK_RABBITMQ_STORAGE_ARGUMENT(2, rabbitmq_exchange_name) - CHECK_RABBITMQ_STORAGE_ARGUMENT(3, rabbitmq_format) + CHECK_RABBITMQ_STORAGE_ARGUMENT(2, rabbitmq_format) + CHECK_RABBITMQ_STORAGE_ARGUMENT(3, rabbitmq_exchange_name) CHECK_RABBITMQ_STORAGE_ARGUMENT(4, rabbitmq_exchange_type) CHECK_RABBITMQ_STORAGE_ARGUMENT(5, rabbitmq_routing_key_list) CHECK_RABBITMQ_STORAGE_ARGUMENT(6, rabbitmq_row_delimiter) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 664959668b3..0e47a8e1150 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -148,7 +148,7 @@ private: void loopingFunc(); void connectionFunc(); - static Names parseSettings(String routing_key_list); + static Names parseSettings(String settings_list); static AMQP::ExchangeType defineExchangeType(String exchange_type_); static String getTableBasedName(String name, const StorageID & table_id); diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index f6274db3d4c..008f5313e22 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -2082,7 +2082,6 @@ def test_rabbitmq_queue_consume(rabbitmq_cluster): CREATE TABLE test.rabbitmq_queue (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', - rabbitmq_exchange_name = 'drop', rabbitmq_format = 'JSONEachRow', rabbitmq_queue_base = 'rabbit_queue', rabbitmq_queue_consume = 1; From 8703b7863dd043c91520411fb1993a22444c7ada Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 5 May 2021 23:56:14 +0300 Subject: [PATCH 012/202] Fix distributed_group_by_no_merge=2 with GROUP BY Before this patch the following query: SELECT assumeNotNull(argMax(dummy, 1)) FROM remote('127.1', system.one) SETTINGS distributed_group_by_no_merge = 2 Leads to: Code: 10. DB::Exception: Received from localhost:9000. DB::Exception: Not found column argMax(dummy, 1) in block: while executing 'INPUT : 0 -> argMax(dummy, 1) UInt8 : 0'. Since it tries to execute function one more time, but shards will not send this column when the query processed with distributed_group_by_no_merge=2 (i.e. up to WithMergeableStateAfterAggregation). v0: no exception v2: execut window functions v3: throw exception, since executing window function in this case will lead to messy output --- src/Interpreters/InterpreterSelectQuery.cpp | 7 ++++++- .../00184_shard_distributed_group_by_no_merge.reference | 2 ++ .../00184_shard_distributed_group_by_no_merge.sql | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 16c9731a427..08077bf8e73 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1249,7 +1249,12 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu // 4) preliminary distinct. // Some of these were already executed at the shards (first_stage), // see the counterpart code and comments there. - if (expressions.need_aggregate) + if (from_aggregation_stage) + { + if (query_analyzer->hasWindow()) + throw Exception("Window functions does not support processing from WithMergeableStateAfterAggregation", ErrorCodes::NOT_IMPLEMENTED); + } + else if (expressions.need_aggregate) { executeExpression(query_plan, expressions.before_window, "Before window functions"); diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference index 453c7fb5af0..b667c57a14c 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.reference @@ -39,3 +39,5 @@ GROUP BY w/ ALIAS 1 ORDER BY w/ ALIAS 0 +func(aggregate function) GROUP BY +0 diff --git a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql index 9912e083777..cce10312e8f 100644 --- a/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql +++ b/tests/queries/0_stateless/00184_shard_distributed_group_by_no_merge.sql @@ -39,4 +39,7 @@ SELECT n FROM remote('127.0.0.{2,3}', currentDatabase(), data_00184) GROUP BY nu SELECT 'ORDER BY w/ ALIAS'; SELECT n FROM remote('127.0.0.{2,3}', currentDatabase(), data_00184) ORDER BY number AS n LIMIT 1 SETTINGS distributed_group_by_no_merge=2; +SELECT 'func(aggregate function) GROUP BY'; +SELECT assumeNotNull(argMax(dummy, 1)) FROM remote('127.1', system.one) SETTINGS distributed_group_by_no_merge=2; + drop table data_00184; From e5cff95cc86abc61a8fe4eb3864cd804d11c5b26 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 5 May 2021 23:56:14 +0300 Subject: [PATCH 013/202] Avoid deadlock due to Log usage in 01568_window_functions_distributed --- .../0_stateless/01568_window_functions_distributed.reference | 2 +- .../queries/0_stateless/01568_window_functions_distributed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01568_window_functions_distributed.reference b/tests/queries/0_stateless/01568_window_functions_distributed.reference index 29d3e5ea885..2de3bd99ea2 100644 --- a/tests/queries/0_stateless/01568_window_functions_distributed.reference +++ b/tests/queries/0_stateless/01568_window_functions_distributed.reference @@ -10,7 +10,7 @@ select max(identity(dummy + 1)) over () from remote('127.0.0.{1,2}', system, one 1 1 drop table if exists t_01568; -create table t_01568 engine Log as select intDiv(number, 3) p, number from numbers(9); +create table t_01568 engine Memory as select intDiv(number, 3) p, number from numbers(9); select sum(number) over w, max(number) over w from t_01568 window w as (partition by p); 3 2 3 2 diff --git a/tests/queries/0_stateless/01568_window_functions_distributed.sql b/tests/queries/0_stateless/01568_window_functions_distributed.sql index 7d9d1ea5c92..7c1a91a9c35 100644 --- a/tests/queries/0_stateless/01568_window_functions_distributed.sql +++ b/tests/queries/0_stateless/01568_window_functions_distributed.sql @@ -9,7 +9,7 @@ select max(identity(dummy + 1)) over () from remote('127.0.0.{1,2}', system, one drop table if exists t_01568; -create table t_01568 engine Log as select intDiv(number, 3) p, number from numbers(9); +create table t_01568 engine Memory as select intDiv(number, 3) p, number from numbers(9); select sum(number) over w, max(number) over w from t_01568 window w as (partition by p); From 6ac50ab9c5d570b9806d87392e2fa4caa6dba597 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 5 May 2021 23:56:15 +0300 Subject: [PATCH 014/202] Extend 01568_window_functions_distributed for distributed_group_by_no_merge --- .../01568_window_functions_distributed.reference | 8 ++++++++ .../0_stateless/01568_window_functions_distributed.sql | 2 ++ 2 files changed, 10 insertions(+) diff --git a/tests/queries/0_stateless/01568_window_functions_distributed.reference b/tests/queries/0_stateless/01568_window_functions_distributed.reference index 2de3bd99ea2..7d5a95046f7 100644 --- a/tests/queries/0_stateless/01568_window_functions_distributed.reference +++ b/tests/queries/0_stateless/01568_window_functions_distributed.reference @@ -49,4 +49,12 @@ select groupArray(groupArray(number)) over (rows unbounded preceding) from remot [[0,3,6,0,3,6]] [[0,3,6,0,3,6],[1,4,7,1,4,7]] [[0,3,6,0,3,6],[1,4,7,1,4,7],[2,5,8,2,5,8]] +select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=1; +[[0,3,6]] +[[0,3,6],[1,4,7]] +[[0,3,6],[1,4,7],[2,5,8]] +[[0,3,6]] +[[0,3,6],[1,4,7]] +[[0,3,6],[1,4,7],[2,5,8]] +select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=2; -- { serverError 48 } drop table t_01568; diff --git a/tests/queries/0_stateless/01568_window_functions_distributed.sql b/tests/queries/0_stateless/01568_window_functions_distributed.sql index 7c1a91a9c35..bc82e1ed6ac 100644 --- a/tests/queries/0_stateless/01568_window_functions_distributed.sql +++ b/tests/queries/0_stateless/01568_window_functions_distributed.sql @@ -19,5 +19,7 @@ select distinct sum(number) over w, max(number) over w from remote('127.0.0.{1,2 -- window functions + aggregation w/shards select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3); +select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=1; +select groupArray(groupArray(number)) over (rows unbounded preceding) from remote('127.0.0.{1,2}', '', t_01568) group by mod(number, 3) settings distributed_group_by_no_merge=2; -- { serverError 48 } drop table t_01568; From eefd67fce592dc727761e661b817b985d4374184 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 6 May 2021 00:26:14 +0300 Subject: [PATCH 015/202] Disable optimize_distributed_group_by_sharding_key with window functions --- src/Interpreters/InterpreterSelectQuery.cpp | 1 + src/Storages/SelectQueryInfo.h | 3 +++ src/Storages/StorageDistributed.cpp | 10 +++++++--- ...ptimize_distributed_group_by_sharding_key.reference | 3 +++ ...1244_optimize_distributed_group_by_sharding_key.sql | 4 ++++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 08077bf8e73..be872a232c3 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -578,6 +578,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl() OpenTelemetrySpanHolder span(__PRETTY_FUNCTION__); query_info.query = query_ptr; + query_info.has_window = query_analyzer->hasWindow(); if (storage && !options.only_analyze) from_stage = storage->getQueryProcessingStage(context, options.to_stage, query_info); diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index b4ac07c612a..7d33172b703 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -139,6 +139,9 @@ struct SelectQueryInfo /// Example: x IN (1, 2, 3) PreparedSets sets; + /// Cached value of ExpressionAnalysisResult::has_window + bool has_window = false; + ClusterPtr getCluster() const { return !optimized_cluster ? cluster : optimized_cluster; } }; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index a402c3e0218..718a15888ea 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -285,9 +285,9 @@ void replaceConstantExpressions( /// - QueryProcessingStage::Complete /// - QueryProcessingStage::WithMergeableStateAfterAggregation /// - none (in this case regular WithMergeableState should be used) -std::optional getOptimizedQueryProcessingStage(const ASTPtr & query_ptr, bool extremes, const Block & sharding_key_block) +std::optional getOptimizedQueryProcessingStage(const SelectQueryInfo & query_info, bool extremes, const Block & sharding_key_block) { - const auto & select = query_ptr->as(); + const auto & select = query_info.query->as(); auto sharding_block_has = [&](const auto & exprs, size_t limit = SIZE_MAX) -> bool { @@ -314,6 +314,10 @@ std::optional getOptimizedQueryProcessingStage(const if (select.group_by_with_totals || select.group_by_with_rollup || select.group_by_with_cube) return {}; + // Window functions are not supported. + if (query_info.has_window) + return {}; + // TODO: extremes support can be implemented if (extremes) return {}; @@ -510,7 +514,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( (settings.allow_nondeterministic_optimize_skip_unused_shards || sharding_key_is_deterministic)) { Block sharding_key_block = sharding_key_expr->getSampleBlock(); - auto stage = getOptimizedQueryProcessingStage(query_info.query, settings.extremes, sharding_key_block); + auto stage = getOptimizedQueryProcessingStage(query_info, settings.extremes, sharding_key_block); if (stage) { LOG_DEBUG(log, "Force processing stage to {}", QueryProcessingStage::toString(*stage)); diff --git a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference index d1697bd2310..acaf6531101 100644 --- a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference +++ b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.reference @@ -123,3 +123,6 @@ GROUP BY sharding_key, ... GROUP BY ..., sharding_key 0 0 1 0 +window functions +0 0 +1 0 diff --git a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql index 2f77155cc54..6b6300a4871 100644 --- a/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql +++ b/tests/queries/0_stateless/01244_optimize_distributed_group_by_sharding_key.sql @@ -106,5 +106,9 @@ select * from dist_01247 group by key, value; select 'GROUP BY ..., sharding_key'; select * from dist_01247 group by value, key; +-- window functions +select 'window functions'; +select key, sum(sum(value)) over (rows unbounded preceding) from dist_01247 group by key settings allow_experimental_window_functions=1; + drop table dist_01247; drop table data_01247; From 308b964ca4f472e3f748f4dbdd58b4f4218119c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Mon, 10 May 2021 20:30:52 +0800 Subject: [PATCH 016/202] enable DateTime64 to be version column --- src/DataTypes/DataTypeDateTime64.h | 2 ++ src/DataTypes/IDataType.h | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index f51e0f5d047..ac0f49613a2 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -35,6 +35,8 @@ public: bool canBePromoted() const override { return false; } + bool canBeUsedAsVersion() const override { return true; } + protected: SerializationPtr doGetDefaultSerialization() const override; }; diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 5a676819b77..fdf4973fce9 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -177,7 +177,7 @@ public: */ virtual bool canBeComparedWithCollation() const { return false; } - /** If the type is totally comparable (Ints, Date, DateTime, not nullable, not floats) + /** If the type is totally comparable (Ints, Date, DateTime, DateTime64, not nullable, not floats) * and "simple" enough (not String, FixedString) to be used as version number * (to select rows with maximum version). */ diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 854b64181cc..22c8b1e5978 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -630,7 +630,7 @@ void MergeTreeData::MergingParams::check(const StorageInMemoryMetadata & metadat throw Exception("The column " + version_column + " cannot be used as a version column for storage " + storage + " because it is of type " + column.type->getName() + - " (must be of an integer type or of type Date or DateTime)", ErrorCodes::BAD_TYPE_OF_FIELD); + " (must be of an integer type or of type Date/DateTime/DateTime64)", ErrorCodes::BAD_TYPE_OF_FIELD); miss_column = false; break; } From 04f00b2b42465092f0d710dc3717103ba45382b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BA=B7?= Date: Tue, 11 May 2021 20:00:34 +0800 Subject: [PATCH 017/202] add tests --- ...67_support_datetime64_version_column.reference | 4 ++++ .../01867_support_datetime64_version_column.sql | 15 +++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/queries/0_stateless/01867_support_datetime64_version_column.reference create mode 100644 tests/queries/0_stateless/01867_support_datetime64_version_column.sql diff --git a/tests/queries/0_stateless/01867_support_datetime64_version_column.reference b/tests/queries/0_stateless/01867_support_datetime64_version_column.reference new file mode 100644 index 00000000000..f449cef23d3 --- /dev/null +++ b/tests/queries/0_stateless/01867_support_datetime64_version_column.reference @@ -0,0 +1,4 @@ +1 1970-01-01 08:25:46.300800003 a1 +2 1970-01-01 08:25:46.300800004 b1 +3 1970-01-01 08:25:46.300800005 c1 +2 1970-01-01 08:25:46.300800005 a1 diff --git a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql new file mode 100644 index 00000000000..d497cf1b2f5 --- /dev/null +++ b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql @@ -0,0 +1,15 @@ +create table replacing( `A` Int64, `D` DateTime64(9), `S` String) ENGINE = ReplacingMergeTree(D) ORDER BY A; + +insert into replacing values (1,'1970-01-01 08:25:46.300800000','a'); +insert into replacing values (2,'1970-01-01 08:25:46.300800002','b'); +insert into replacing values (1,'1970-01-01 08:25:46.300800003','a1'); +insert into replacing values (1,'1970-01-01 08:25:46.300800002','a2'); +insert into replacing values (2,'1970-01-01 08:25:46.300800004','b1'); +insert into replacing values (3,'1970-01-01 08:25:46.300800005','c1'); +insert into replacing values (2,'1970-01-01 08:25:46.300800005','a1'); + +OPTIMIZE TABLE replacing; + +select * from replacing; + +drop table replacing; \ No newline at end of file From 3bb480bd376ee8bdf9090fc8ea9e26d46a6aeb5c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 11 May 2021 15:29:37 +0300 Subject: [PATCH 018/202] Update 01867_support_datetime64_version_column.sql --- .../0_stateless/01867_support_datetime64_version_column.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql index d497cf1b2f5..9e7d487ea74 100644 --- a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql +++ b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql @@ -1,3 +1,4 @@ +drop table if exists replacing; create table replacing( `A` Int64, `D` DateTime64(9), `S` String) ENGINE = ReplacingMergeTree(D) ORDER BY A; insert into replacing values (1,'1970-01-01 08:25:46.300800000','a'); @@ -12,4 +13,4 @@ OPTIMIZE TABLE replacing; select * from replacing; -drop table replacing; \ No newline at end of file +drop table replacing; From f6c9327fd55c9136d4a8d04277bbfbccefb7a0bd Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 11 May 2021 15:29:48 +0300 Subject: [PATCH 019/202] Update 01867_support_datetime64_version_column.sql --- .../0_stateless/01867_support_datetime64_version_column.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql index 9e7d487ea74..f4427be635a 100644 --- a/tests/queries/0_stateless/01867_support_datetime64_version_column.sql +++ b/tests/queries/0_stateless/01867_support_datetime64_version_column.sql @@ -9,7 +9,7 @@ insert into replacing values (2,'1970-01-01 08:25:46.300800004','b1'); insert into replacing values (3,'1970-01-01 08:25:46.300800005','c1'); insert into replacing values (2,'1970-01-01 08:25:46.300800005','a1'); -OPTIMIZE TABLE replacing; +OPTIMIZE TABLE replacing FINAL; select * from replacing; From e8524d6e69eeb831e319c9d21e86de58ddcad56c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 12 May 2021 01:19:51 +0300 Subject: [PATCH 020/202] Update 01867_support_datetime64_version_column.reference --- .../01867_support_datetime64_version_column.reference | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01867_support_datetime64_version_column.reference b/tests/queries/0_stateless/01867_support_datetime64_version_column.reference index f449cef23d3..3a6dbdc870c 100644 --- a/tests/queries/0_stateless/01867_support_datetime64_version_column.reference +++ b/tests/queries/0_stateless/01867_support_datetime64_version_column.reference @@ -1,4 +1,3 @@ 1 1970-01-01 08:25:46.300800003 a1 -2 1970-01-01 08:25:46.300800004 b1 -3 1970-01-01 08:25:46.300800005 c1 2 1970-01-01 08:25:46.300800005 a1 +3 1970-01-01 08:25:46.300800005 c1 From ffb4e1af8f9180b00d99c8a63d13278830377321 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 13:39:07 +0300 Subject: [PATCH 021/202] Some copy-paste --- programs/CMakeLists.txt | 26 +- programs/config_tools.h.in | 1 + programs/keeper/CMakeLists.txt | 14 + programs/keeper/Keeper.cpp | 421 ++++++++++++++++++++++++++ programs/keeper/Keeper.h | 64 ++++ programs/keeper/clickhouse-keeper.cpp | 6 + programs/main.cpp | 6 + 7 files changed, 534 insertions(+), 4 deletions(-) create mode 100644 programs/keeper/CMakeLists.txt create mode 100644 programs/keeper/Keeper.cpp create mode 100644 programs/keeper/Keeper.h create mode 100644 programs/keeper/clickhouse-keeper.cpp diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 09199e83026..500e4794335 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -47,6 +47,9 @@ option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Li option (ENABLE_CLICKHOUSE_GIT_IMPORT "A tool to analyze Git repositories" ${ENABLE_CLICKHOUSE_ALL}) + +option (ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ENABLE_CLICKHOUSE_ALL}) + if (CLICKHOUSE_SPLIT_BINARY) option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" OFF) else () @@ -134,6 +137,12 @@ else() message(STATUS "ClickHouse git-import: OFF") endif() +if (ENABLE_CLICKHOUSE_KEEPER) + message(STATUS "ClickHouse keeper mode: ON") +else() + message(STATUS "ClickHouse keeper mode: OFF") +endif() + if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES)) set(CLICKHOUSE_ONE_SHARED ON) endif() @@ -202,6 +211,7 @@ add_subdirectory (obfuscator) add_subdirectory (install) add_subdirectory (git-import) add_subdirectory (bash-completion) +add_subdirectory (keeper) if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) add_subdirectory (odbc-bridge) @@ -212,15 +222,15 @@ if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE) endif () if (CLICKHOUSE_ONE_SHARED) - add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES}) - target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK}) - target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_GIT_IMPORT_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE}) + add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES} ${CLICKHOUSE_KEEPER_SOURCES}) + target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK} ${CLICKHOUSE_KEEPER_LINK}) + target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_GIT_IMPORT_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} ${CLICKHOUSE_KEEPER_INCLUDE}) set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse DEBUG_POSTFIX "") install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) endif() if (CLICKHOUSE_SPLIT_BINARY) - set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-git-import clickhouse-copier) + set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-obfuscator clickhouse-git-import clickhouse-copier clickhouse-keeper) if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge) @@ -277,6 +287,9 @@ else () if (ENABLE_CLICKHOUSE_GIT_IMPORT) clickhouse_target_link_split_lib(clickhouse git-import) endif () + if (ENABLE_CLICKHOUSE_KEEPER) + clickhouse_target_link_split_lib(clickhouse keeper) + endif() if (ENABLE_CLICKHOUSE_INSTALL) clickhouse_target_link_split_lib(clickhouse install) endif () @@ -332,6 +345,11 @@ else () install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import) endif () + if (ENABLE_CLICKHOUSE_KEEPER) + add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper) + endif () install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) diff --git a/programs/config_tools.h.in b/programs/config_tools.h.in index abe9ef8c562..50ba0c16a83 100644 --- a/programs/config_tools.h.in +++ b/programs/config_tools.h.in @@ -16,3 +16,4 @@ #cmakedefine01 ENABLE_CLICKHOUSE_INSTALL #cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE #cmakedefine01 ENABLE_CLICKHOUSE_LIBRARY_BRIDGE +#cmakedefine01 ENABLE_CLICKHOUSE_KEEPER diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt new file mode 100644 index 00000000000..8f40fe54667 --- /dev/null +++ b/programs/keeper/CMakeLists.txt @@ -0,0 +1,14 @@ +set(CLICKHOUSE_KEEPER_SOURCES + Keeper.cpp +) + +set (CLICKHOUSE_KEEPER_LINK + PRIVATE + clickhouse_common_config + clickhouse_common_io + clickhouse_common_zookeeper + daemon + dbms +) + +clickhouse_program_add(keeper) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp new file mode 100644 index 00000000000..78c9e7d50b9 --- /dev/null +++ b/programs/keeper/Keeper.cpp @@ -0,0 +1,421 @@ +#include "Keeper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +# include "Common/config_version.h" +#endif + +#if USE_SSL +# include +# include +#endif + +#if USE_NURAFT +# include +#endif + +int mainEntryClickHouseKeeper(int argc, char ** argv) +{ + DB::Keeper app; + + try + { + return app.run(argc, argv); + } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + auto code = DB::getCurrentExceptionCode(); + return code ? code : 1; + } +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NO_ELEMENTS_IN_CONFIG; + extern const int SUPPORT_IS_DISABLED; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int EXCESSIVE_ELEMENT_IN_CONFIG; + extern const int INVALID_CONFIG_PARAMETER; + extern const int SYSTEM_ERROR; + extern const int FAILED_TO_GETPWUID; + extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; + extern const int NETWORK_ERROR; + extern const int CORRUPTED_DATA; +} + +namespace +{ + +int waitServersToFinish(std::vector & servers, size_t seconds_to_wait) +{ + const int sleep_max_ms = 1000 * seconds_to_wait; + const int sleep_one_ms = 100; + int sleep_current_ms = 0; + int current_connections = 0; + for (;;) + { + current_connections = 0; + + for (auto & server : servers) + { + server.stop(); + current_connections += server.currentConnections(); + } + + if (!current_connections) + break; + + sleep_current_ms += sleep_one_ms; + if (sleep_current_ms < sleep_max_ms) + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_one_ms)); + else + break; + } + return current_connections; +} + +Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log) +{ + Poco::Net::SocketAddress socket_address; + try + { + socket_address = Poco::Net::SocketAddress(host, port); + } + catch (const Poco::Net::DNSException & e) + { + const auto code = e.code(); + if (code == EAI_FAMILY +#if defined(EAI_ADDRFAMILY) + || code == EAI_ADDRFAMILY +#endif + ) + { + LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. " + "If it is an IPv6 address and your host has disabled IPv6, then consider to " + "specify IPv4 address to listen in element of configuration " + "file. Example: 0.0.0.0", + host, e.code(), e.message()); + } + + throw; + } + return socket_address; +} + +std::string getCanonicalPath(std::string && path) +{ + Poco::trimInPlace(path); + if (path.empty()) + throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER); + if (path.back() != '/') + path += '/'; + return std::move(path); +} + +[[noreturn]] void forceShutdown() +{ +#if defined(THREAD_SANITIZER) && defined(OS_LINUX) + /// Thread sanitizer tries to do something on exit that we don't need if we want to exit immediately, + /// while connection handling threads are still run. + (void)syscall(SYS_exit_group, 0); + __builtin_unreachable(); +#else + _exit(0); +#endif +} + +} + + +Poco::Net::SocketAddress Keeper::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) const +{ + auto address = makeSocketAddress(host, port, &logger()); +#if !defined(POCO_CLICKHOUSE_PATCH) || POCO_VERSION < 0x01090100 + if (secure) + /// Bug in old (<1.9.1) poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl + /// https://github.com/pocoproject/poco/pull/2257 + socket.bind(address, /* reuseAddress = */ true); + else +#endif +#if POCO_VERSION < 0x01080000 + socket.bind(address, /* reuseAddress = */ true); +#else + socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false)); +#endif + + socket.listen(/* backlog = */ config().getUInt("listen_backlog", 64)); + + return address; +} + +void Keeper::createServer(const std::string & listen_host, const char * port_name, bool listen_try, CreateServerFunc && func) const +{ + /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file. + if (!config().has(port_name)) + return; + + auto port = config().getInt(port_name); + try + { + func(port); + } + catch (const Poco::Exception &) + { + std::string message = "Listen [" + listen_host + "]:" + std::to_string(port) + " failed: " + getCurrentExceptionMessage(false); + + if (listen_try) + { + LOG_WARNING(&logger(), "{}. If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to " + "specify not disabled IPv4 or IPv6 address to listen in element of configuration " + "file. Example for disabled IPv6: 0.0.0.0 ." + " Example for disabled IPv4: ::", + message); + } + else + { + throw Exception{message, ErrorCodes::NETWORK_ERROR}; + } + } +} + +void Keeper::uninitialize() +{ + logger().information("shutting down"); + BaseDaemon::uninitialize(); +} + +int Keeper::run() +{ + if (config().hasOption("help")) + { + Poco::Util::HelpFormatter help_formatter(Keeper::options()); + auto header_str = fmt::format("{} [OPTION] [-- [ARG]...]\n" + "positional arguments can be used to rewrite config.xml properties, for example, --http_port=8010", + commandName()); + help_formatter.setHeader(header_str); + help_formatter.format(std::cout); + return 0; + } + if (config().hasOption("version")) + { + std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; + return 0; + } + + + return Application::run(); // NOLINT +} + +void Keeper::initialize(Poco::Util::Application & self) +{ + BaseDaemon::initialize(self); + logger().information("starting up"); + + LOG_INFO(&logger(), "OS Name = {}, OS Version = {}, OS Architecture = {}", + Poco::Environment::osName(), + Poco::Environment::osVersion(), + Poco::Environment::osArchitecture()); +} + +std::string Keeper::getDefaultCorePath() const +{ + return getCanonicalPath(config().getString("path", KEEPER_DEFAULT_PATH)) + "cores"; +} + +void Keeper::defineOptions(Poco::Util::OptionSet & options) +{ + options.addOption( + Poco::Util::Option("help", "h", "show help and exit") + .required(false) + .repeatable(false) + .binding("help")); + options.addOption( + Poco::Util::Option("version", "V", "show version and exit") + .required(false) + .repeatable(false) + .binding("version")); + BaseDaemon::defineOptions(options); +} + +int Keeper::main(const std::vector & /*args*/) +{ + Poco::Logger * log = &logger(); + + UseSSL use_ssl; + + MainThreadStatus::getInstance(); + +#if !defined(NDEBUG) || !defined(__OPTIMIZE__) + LOG_WARNING(log, "Server was built in debug mode. It will work slowly."); +#endif + +#if defined(SANITIZER) + LOG_WARNING(log, "Server was built with sanitizer. It will work slowly."); +#endif + + auto shared_context = Context::createShared(); + global_context = Context::createGlobal(shared_context.get()); + + global_context->makeGlobalContext(); + global_context->setApplicationType(Context::ApplicationType::SERVER); + const Settings & settings = global_context->getSettingsRef(); + + GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 500)); + + static ServerErrorHandler error_handler; + Poco::ErrorHandler::set(&error_handler); + + /// Initialize DateLUT early, to not interfere with running time of first query. + LOG_DEBUG(log, "Initializing DateLUT."); + DateLUT::instance(); + LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); + + Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); + + Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); + + std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); + + bool listen_try = config().getBool("listen_try", false); + if (listen_hosts.empty()) + { + listen_hosts.emplace_back("::1"); + listen_hosts.emplace_back("127.0.0.1"); + listen_try = true; + } + + auto servers = std::make_shared>(); + + if (config().has("keeper_server")) + { +#if USE_NURAFT + /// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config. + global_context->initializeKeeperStorageDispatcher(); + for (const auto & listen_host : listen_hosts) + { + /// TCP Keeper + const char * port_name = "keeper_server.tcp_port"; + createServer(listen_host, port_name, listen_try, [&](UInt16 port) + { + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, listen_host, port); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); + servers->emplace_back( + port_name, + std::make_unique( + new KeeperTCPHandlerFactory(*this, false), server_pool, socket, new Poco::Net::TCPServerParams)); + + LOG_INFO(log, "Listening for connections to Keeper (tcp): {}", address.toString()); + }); + + const char * secure_port_name = "keeper_server.tcp_port_secure"; + createServer(listen_host, secure_port_name, listen_try, [&](UInt16 port) + { +#if USE_SSL + Poco::Net::SecureServerSocket socket; + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); + servers->emplace_back( + secure_port_name, + std::make_unique( + new KeeperTCPHandlerFactory(*this, true), server_pool, socket, new Poco::Net::TCPServerParams)); + LOG_INFO(log, "Listening for connections to Keeper with secure protocol (tcp_secure): {}", address.toString()); +#else + UNUSED(port); + throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", + ErrorCodes::SUPPORT_IS_DISABLED}; +#endif + }); + } +#else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse server built without NuRaft library. Cannot use internal coordination."); +#endif + + } + for (auto & server : *servers) + server.start(); + + SCOPE_EXIT({ + /** Ask to cancel background jobs all table engines, + * and also query_log. + * It is important to do early, not in destructor of Context, because + * table engines could use Context on destroy. + */ + LOG_INFO(log, "Shutting down."); + + global_context->shutdown(); + + LOG_DEBUG(log, "Waiting for current connections to servers for tables to finish."); + int current_connections = 0; + for (auto & server : *servers) + { + server.stop(); + current_connections += server.currentConnections(); + } + + if (current_connections) + LOG_INFO(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections); + else + LOG_INFO(log, "Closed all listening sockets."); + + if (current_connections > 0) + current_connections = waitServersToFinish(*servers, config().getInt("shutdown_wait_unfinished", 5)); + + if (current_connections) + LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections); + else + LOG_INFO(log, "Closed connections to servers for tables."); + + global_context->shutdownKeeperStorageDispatcher(); + + /// Wait server pool to avoid use-after-free of destroyed context in the handlers + server_pool.joinAll(); + + /** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available. + * At this moment, no one could own shared part of Context. + */ + global_context.reset(); + shared_context.reset(); + + LOG_DEBUG(log, "Destroyed global context."); + + if (current_connections) + { + LOG_INFO(log, "Will shutdown forcefully."); + forceShutdown(); + } + }); + + + buildLoggers(config(), logger()); + + LOG_INFO(log, "Ready for connections."); + + waitForTerminationRequest(); + + return Application::EXIT_OK; +} + +} diff --git a/programs/keeper/Keeper.h b/programs/keeper/Keeper.h new file mode 100644 index 00000000000..0fe3465f789 --- /dev/null +++ b/programs/keeper/Keeper.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +namespace Poco +{ + namespace Net + { + class ServerSocket; + } +} + +namespace DB +{ + +class Keeper : public BaseDaemon, public IServer +{ +public: + using ServerApplication::run; + + Poco::Util::LayeredConfiguration & config() const override + { + return BaseDaemon::config(); + } + + Poco::Logger & logger() const override + { + return BaseDaemon::logger(); + } + + ContextPtr context() const override + { + return global_context; + } + + bool isCancelled() const override + { + return BaseDaemon::isCancelled(); + } + + void defineOptions(Poco::Util::OptionSet & _options) override; + +protected: + int run() override; + + void initialize(Application & self) override; + + void uninitialize() override; + + int main(const std::vector & args) override; + + std::string getDefaultCorePath() const override; + +private: + ContextPtr global_context; + + Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure = false) const; + + using CreateServerFunc = std::function; + void createServer(const std::string & listen_host, const char * port_name, bool listen_try, CreateServerFunc && func) const; +}; + +} diff --git a/programs/keeper/clickhouse-keeper.cpp b/programs/keeper/clickhouse-keeper.cpp new file mode 100644 index 00000000000..baa673f79ee --- /dev/null +++ b/programs/keeper/clickhouse-keeper.cpp @@ -0,0 +1,6 @@ +int mainEntryClickHouseKeeper(int argc, char ** argv); + +int main(int argc_, char ** argv_) +{ + return mainEntryClickHouseKeeper(argc_, argv_); +} diff --git a/programs/main.cpp b/programs/main.cpp index cbb22b7a87b..ccdf4d50fb4 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -55,6 +55,9 @@ int mainEntryClickHouseObfuscator(int argc, char ** argv); #if ENABLE_CLICKHOUSE_GIT_IMPORT int mainEntryClickHouseGitImport(int argc, char ** argv); #endif +#if ENABLE_CLICKHOUSE_KEEPER +int mainEntryClickHouseKeeper(int argc, char ** argv); +#endif #if ENABLE_CLICKHOUSE_INSTALL int mainEntryClickHouseInstall(int argc, char ** argv); int mainEntryClickHouseStart(int argc, char ** argv); @@ -112,6 +115,9 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_GIT_IMPORT {"git-import", mainEntryClickHouseGitImport}, #endif +#if ENABLE_CLICKHOUSE_KEEPER + {"keeper", mainEntryClickHouseKeeper}, +#endif #if ENABLE_CLICKHOUSE_INSTALL {"install", mainEntryClickHouseInstall}, {"start", mainEntryClickHouseStart}, From 41eadf9127cb8dda4153c8d28e87f6fba6a394ad Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 16:04:34 +0300 Subject: [PATCH 022/202] Something working --- base/daemon/BaseDaemon.cpp | 7 +- base/daemon/BaseDaemon.h | 2 + programs/keeper/CMakeLists.txt | 2 + programs/keeper/Keeper.cpp | 115 +++++++++++++------------- programs/keeper/Keeper.h | 4 +- programs/server/CMakeLists.txt | 2 +- src/Common/Config/ConfigProcessor.cpp | 13 ++- src/Core/Defines.h | 2 + 8 files changed, 85 insertions(+), 62 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 83384038b7c..01e700ebba3 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -468,7 +468,7 @@ void BaseDaemon::reloadConfiguration() * instead of using files specified in config.xml. * (It's convenient to log in console when you start server without any command line parameters.) */ - config_path = config().getString("config-file", "config.xml"); + config_path = config().getString("config-file", getDefaultConfigFileName()); DB::ConfigProcessor config_processor(config_path, false, true); config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true); @@ -516,6 +516,11 @@ std::string BaseDaemon::getDefaultCorePath() const return "/opt/cores/"; } +std::string BaseDaemon::getDefaultConfigFileName() const +{ + return "config.xml"; +} + void BaseDaemon::closeFDs() { #if defined(OS_FREEBSD) || defined(OS_DARWIN) diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 8b9d765cf2e..3d47d85a9f5 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -149,6 +149,8 @@ protected: virtual std::string getDefaultCorePath() const; + virtual std::string getDefaultConfigFileName() const; + std::optional pid_file; std::atomic_bool is_cancelled{false}; diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 8f40fe54667..cd10baa2e57 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -12,3 +12,5 @@ set (CLICKHOUSE_KEEPER_LINK ) clickhouse_program_add(keeper) + +install (FILES clickhouse-keeper.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper RENAME keeper_config.xml) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 78c9e7d50b9..e64134ed10d 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -1,5 +1,6 @@ #include "Keeper.h" +#include #include #include #include @@ -122,16 +123,6 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port return socket_address; } -std::string getCanonicalPath(std::string && path) -{ - Poco::trimInPlace(path); - if (path.empty()) - throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER); - if (path.back() != '/') - path += '/'; - return std::move(path); -} - [[noreturn]] void forceShutdown() { #if defined(THREAD_SANITIZER) && defined(OS_LINUX) @@ -146,7 +137,6 @@ std::string getCanonicalPath(std::string && path) } - Poco::Net::SocketAddress Keeper::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) const { auto address = makeSocketAddress(host, port, &logger()); @@ -218,11 +208,10 @@ int Keeper::run() } if (config().hasOption("version")) { - std::cout << DBMS_NAME << " server version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; + std::cout << DBMS_NAME << " keeper version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; return 0; } - return Application::run(); // NOLINT } @@ -237,9 +226,9 @@ void Keeper::initialize(Poco::Util::Application & self) Poco::Environment::osArchitecture()); } -std::string Keeper::getDefaultCorePath() const +std::string Keeper::getDefaultConfigFileName() const { - return getCanonicalPath(config().getString("path", KEEPER_DEFAULT_PATH)) + "cores"; + return "keeper_config.xml"; } void Keeper::defineOptions(Poco::Util::OptionSet & options) @@ -266,11 +255,11 @@ int Keeper::main(const std::vector & /*args*/) MainThreadStatus::getInstance(); #if !defined(NDEBUG) || !defined(__OPTIMIZE__) - LOG_WARNING(log, "Server was built in debug mode. It will work slowly."); + LOG_WARNING(log, "Keeper was built in debug mode. It will work slowly."); #endif #if defined(SANITIZER) - LOG_WARNING(log, "Server was built with sanitizer. It will work slowly."); + LOG_WARNING(log, "Keeper was built with sanitizer. It will work slowly."); #endif auto shared_context = Context::createShared(); @@ -278,6 +267,10 @@ int Keeper::main(const std::vector & /*args*/) global_context->makeGlobalContext(); global_context->setApplicationType(Context::ApplicationType::SERVER); + + if (!config().has("keeper_server")) + throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Keeper configuration ( section) not found in config"); + const Settings & settings = global_context->getSettingsRef(); GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 500)); @@ -290,9 +283,10 @@ int Keeper::main(const std::vector & /*args*/) DateLUT::instance(); LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::instance().getTimeZone()); - Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); + /// Don't want to use DNS cache + DNSResolver::instance().setDisableCacheFlag(); - Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); + Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); @@ -306,54 +300,51 @@ int Keeper::main(const std::vector & /*args*/) auto servers = std::make_shared>(); - if (config().has("keeper_server")) - { #if USE_NURAFT - /// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config. - global_context->initializeKeeperStorageDispatcher(); - for (const auto & listen_host : listen_hosts) + /// Initialize test keeper RAFT. Do nothing if no nu_keeper_server in config. + global_context->initializeKeeperStorageDispatcher(); + for (const auto & listen_host : listen_hosts) + { + /// TCP Keeper + const char * port_name = "keeper_server.tcp_port"; + createServer(listen_host, port_name, listen_try, [&](UInt16 port) { - /// TCP Keeper - const char * port_name = "keeper_server.tcp_port"; - createServer(listen_host, port_name, listen_try, [&](UInt16 port) - { - Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); - servers->emplace_back( - port_name, - std::make_unique( - new KeeperTCPHandlerFactory(*this, false), server_pool, socket, new Poco::Net::TCPServerParams)); + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, listen_host, port); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); + servers->emplace_back( + port_name, + std::make_unique( + new KeeperTCPHandlerFactory(*this, false), server_pool, socket, new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening for connections to Keeper (tcp): {}", address.toString()); - }); + LOG_INFO(log, "Listening for connections to Keeper (tcp): {}", address.toString()); + }); - const char * secure_port_name = "keeper_server.tcp_port_secure"; - createServer(listen_host, secure_port_name, listen_try, [&](UInt16 port) - { + const char * secure_port_name = "keeper_server.tcp_port_secure"; + createServer(listen_host, secure_port_name, listen_try, [&](UInt16 port) + { #if USE_SSL - Poco::Net::SecureServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); - socket.setReceiveTimeout(settings.receive_timeout); - socket.setSendTimeout(settings.send_timeout); - servers->emplace_back( - secure_port_name, - std::make_unique( - new KeeperTCPHandlerFactory(*this, true), server_pool, socket, new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening for connections to Keeper with secure protocol (tcp_secure): {}", address.toString()); + Poco::Net::SecureServerSocket socket; + auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); + servers->emplace_back( + secure_port_name, + std::make_unique( + new KeeperTCPHandlerFactory(*this, true), server_pool, socket, new Poco::Net::TCPServerParams)); + LOG_INFO(log, "Listening for connections to Keeper with secure protocol (tcp_secure): {}", address.toString()); #else - UNUSED(port); - throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", - ErrorCodes::SUPPORT_IS_DISABLED}; + UNUSED(port); + throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", + ErrorCodes::SUPPORT_IS_DISABLED}; #endif - }); - } + }); + } #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse server built without NuRaft library. Cannot use internal coordination."); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse keeper built without NuRaft library. Cannot use coordination."); #endif - } for (auto & server : *servers) server.start(); @@ -418,4 +409,14 @@ int Keeper::main(const std::vector & /*args*/) return Application::EXIT_OK; } + +void Keeper::logRevision() const +{ + Poco::Logger::root().information("Starting ClickHouse Keeper " + std::string{VERSION_STRING} + + " with revision " + std::to_string(ClickHouseRevision::getVersionRevision()) + + ", " + build_id_info + + ", PID " + std::to_string(getpid())); +} + + } diff --git a/programs/keeper/Keeper.h b/programs/keeper/Keeper.h index 0fe3465f789..ba34dbea313 100644 --- a/programs/keeper/Keeper.h +++ b/programs/keeper/Keeper.h @@ -42,6 +42,8 @@ public: void defineOptions(Poco::Util::OptionSet & _options) override; protected: + void logRevision() const override; + int run() override; void initialize(Application & self) override; @@ -50,7 +52,7 @@ protected: int main(const std::vector & args) override; - std::string getDefaultCorePath() const override; + std::string getDefaultConfigFileName() const override; private: ContextPtr global_context; diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 0dcfbce1c30..bc7f2d94153 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -38,7 +38,7 @@ if (OS_LINUX) # 1. Allow to run the binary without download of any other files. # 2. Allow to implement "sudo clickhouse install" tool. - foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html) + foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html keeper_embedded.xml) set(RESOURCE_OBJ ${RESOURCE_FILE}.o) set(RESOURCE_OBJS ${RESOURCE_OBJS} ${RESOURCE_OBJ}) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 39ab407579d..bc2a8a27943 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -462,10 +462,19 @@ XMLDocumentPtr ConfigProcessor::processConfig( } else { - /// When we can use config embedded in binary. + /// These embedded files added during build with some cmake magic. + /// Look at the end of programs/sever/CMakeLists.txt. + std::string embedded_name; if (path == "config.xml") + embedded_name = "embedded.xml"; + + if (path == "keeper_config.xml") + embedded_name = "keeper_embedded.xml"; + + /// When we can use config embedded in binary. + if (!embedded_name.empty()) { - auto resource = getResource("embedded.xml"); + auto resource = getResource(embedded_name); if (resource.empty()) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Configuration file {} doesn't exist and there is no embedded config", path); LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path); diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 668a60f9be8..fbcc6fabbf0 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -98,6 +98,8 @@ #define DBMS_DEFAULT_PATH "/var/lib/clickhouse/" +#define KEEPER_DEFAULT_PATH "/var/lib/clickhouse/coordination" + // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html /// Marks that extra information is sent to a shard. It could be any magic numbers. From ed89af22b27d14ee3e694bfd7b504921ab58d45e Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 16:31:29 +0300 Subject: [PATCH 023/202] Move jepsen tests to clickhouse-keeper --- .../jepsen.clickhouse-keeper/resources/config.xml | 1 - .../resources/keeper_config.xml | 8 ++++++++ .../jepsen.clickhouse-keeper/resources/listen.xml | 3 --- .../jepsen.clickhouse-keeper/resources/users.xml | 1 - .../src/jepsen/clickhouse_keeper/db.clj | 7 ++----- .../src/jepsen/clickhouse_keeper/utils.clj | 15 ++++++--------- 6 files changed, 16 insertions(+), 19 deletions(-) delete mode 120000 tests/jepsen.clickhouse-keeper/resources/config.xml delete mode 100644 tests/jepsen.clickhouse-keeper/resources/listen.xml delete mode 120000 tests/jepsen.clickhouse-keeper/resources/users.xml diff --git a/tests/jepsen.clickhouse-keeper/resources/config.xml b/tests/jepsen.clickhouse-keeper/resources/config.xml deleted file mode 120000 index c7596baa075..00000000000 --- a/tests/jepsen.clickhouse-keeper/resources/config.xml +++ /dev/null @@ -1 +0,0 @@ -../../../programs/server/config.xml \ No newline at end of file diff --git a/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml b/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml index 528ea5d77be..f06d9683990 100644 --- a/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml +++ b/tests/jepsen.clickhouse-keeper/resources/keeper_config.xml @@ -1,4 +1,12 @@ + :: + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + 9181 {id} diff --git a/tests/jepsen.clickhouse-keeper/resources/listen.xml b/tests/jepsen.clickhouse-keeper/resources/listen.xml deleted file mode 100644 index de8c737ff75..00000000000 --- a/tests/jepsen.clickhouse-keeper/resources/listen.xml +++ /dev/null @@ -1,3 +0,0 @@ - - :: - diff --git a/tests/jepsen.clickhouse-keeper/resources/users.xml b/tests/jepsen.clickhouse-keeper/resources/users.xml deleted file mode 120000 index 41b137a130f..00000000000 --- a/tests/jepsen.clickhouse-keeper/resources/users.xml +++ /dev/null @@ -1 +0,0 @@ -../../../programs/server/users.xml \ No newline at end of file diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj index fdb6b233fec..30c2c0eaf4f 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/db.clj @@ -89,10 +89,7 @@ (defn install-configs [test node] - (c/exec :echo (slurp (io/resource "config.xml")) :> (str configs-dir "/config.xml")) - (c/exec :echo (slurp (io/resource "users.xml")) :> (str configs-dir "/users.xml")) - (c/exec :echo (slurp (io/resource "listen.xml")) :> (str sub-configs-dir "/listen.xml")) - (c/exec :echo (cluster-config test node (slurp (io/resource "keeper_config.xml"))) :> (str sub-configs-dir "/keeper_config.xml"))) + (c/exec :echo (cluster-config test node (slurp (io/resource "keeper_config.xml"))) :> (str configs-dir "/keeper_config.xml"))) (defn collect-traces [test node] @@ -144,7 +141,7 @@ (info node "Coordination files exists, going to compress") (c/cd data-dir (c/exec :tar :czf "coordination.tar.gz" "coordination"))))) - (let [common-logs [stderr-file (str logs-dir "/clickhouse-server.log") (str data-dir "/coordination.tar.gz")] + (let [common-logs [stderr-file (str logs-dir "/clickhouse-keeper.log") (str data-dir "/coordination.tar.gz")] gdb-log (str logs-dir "/gdb.log")] (if (cu/exists? (str logs-dir "/gdb.log")) (conj common-logs gdb-log) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj index 70813457251..0457ff6eae2 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj @@ -143,7 +143,7 @@ [node test] (info "Checking server alive on" node) (try - (c/exec binary-path :client :--query "SELECT 1") + (zk-connect (name node) 9181 30000) (catch Exception _ false))) (defn wait-clickhouse-alive! @@ -169,16 +169,13 @@ :logfile stderr-file :chdir data-dir} binary-path - :server - :--config (str configs-dir "/config.xml") + :keeper + :--config (str configs-dir "/keeper_config.xml") :-- - :--path (str data-dir "/") - :--user_files_path (str data-dir "/user_files") - :--top_level_domains_path (str data-dir "/top_level_domains") - :--logger.log (str logs-dir "/clickhouse-server.log") - :--logger.errorlog (str logs-dir "/clickhouse-server.err.log") + :--logger.log (str logs-dir "/clickhouse-keeper.log") + :--logger.errorlog (str logs-dir "/clickhouse-keeper.err.log") :--keeper_server.snapshot_storage_path coordination-snapshots-dir - :--keeper_server.logs_storage_path coordination-logs-dir) + :--keeper_server.log_storage_path coordination-logs-dir) (wait-clickhouse-alive! node test))) (defn md5 [^String s] From 43ee9f0a3a3b3998a930c92e97ef7b4e43f3b111 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 17:05:44 +0300 Subject: [PATCH 024/202] Check for directory owner --- programs/keeper/Keeper.cpp | 70 ++++++++++++++++++++++--- src/Coordination/KeeperServer.cpp | 15 +++++- src/Coordination/KeeperStateManager.cpp | 20 ++++++- 3 files changed, 96 insertions(+), 9 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index e64134ed10d..a7e90eb5f07 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -1,5 +1,7 @@ #include "Keeper.h" +#include +#include #include #include #include @@ -15,6 +17,7 @@ #include #include #include +#include #include #if !defined(ARCADIA_BUILD) @@ -31,6 +34,11 @@ # include #endif +#if defined(OS_LINUX) +# include +# include +#endif + int mainEntryClickHouseKeeper(int argc, char ** argv) { DB::Keeper app; @@ -54,14 +62,9 @@ namespace ErrorCodes { extern const int NO_ELEMENTS_IN_CONFIG; extern const int SUPPORT_IS_DISABLED; - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int INVALID_CONFIG_PARAMETER; - extern const int SYSTEM_ERROR; - extern const int FAILED_TO_GETPWUID; - extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; extern const int NETWORK_ERROR; - extern const int CORRUPTED_DATA; + extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA; + extern const int FAILED_TO_GETPWUID; } namespace @@ -135,6 +138,26 @@ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port #endif } +std::string getUserName(uid_t user_id) +{ + /// Try to convert user id into user name. + auto buffer_size = sysconf(_SC_GETPW_R_SIZE_MAX); + if (buffer_size <= 0) + buffer_size = 1024; + std::string buffer; + buffer.reserve(buffer_size); + + struct passwd passwd_entry; + struct passwd * result = nullptr; + const auto error = getpwuid_r(user_id, &passwd_entry, buffer.data(), buffer_size, &result); + + if (error) + throwFromErrno("Failed to find user name for " + toString(user_id), ErrorCodes::FAILED_TO_GETPWUID, error); + else if (result) + return result->pw_name; + return toString(user_id); +} + } Poco::Net::SocketAddress Keeper::socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, [[maybe_unused]] bool secure) const @@ -271,6 +294,39 @@ int Keeper::main(const std::vector & /*args*/) if (!config().has("keeper_server")) throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Keeper configuration ( section) not found in config"); + + std::string path; + + if (config().has("keeper_server.storage_path")) + path = config().getString("keeper_server.storage_path"); + else if (config().has("keeper_server.log_storage_path")) + path = config().getString("keeper_server.log_storage_path"); + else if (config().has("keeper_server.snapshot_storage_path")) + path = config().getString("keeper_server.snapshot_storage_path"); + else + path = std::filesystem::path{DBMS_DEFAULT_PATH} / "coordination/logs"; + + + /// Check that the process user id matches the owner of the data. + const auto effective_user_id = geteuid(); + struct stat statbuf; + if (stat(path.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) + { + const auto effective_user = getUserName(effective_user_id); + const auto data_owner = getUserName(statbuf.st_uid); + std::string message = "Effective user of the process (" + effective_user + + ") does not match the owner of the data (" + data_owner + ")."; + if (effective_user_id == 0) + { + message += " Run under 'sudo -u " + data_owner + "'."; + throw Exception(message, ErrorCodes::MISMATCHING_USERS_FOR_PROCESS_AND_DATA); + } + else + { + LOG_WARNING(log, message); + } + } + const Settings & settings = global_context->getSettingsRef(); GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 500)); diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index a3214474e96..56165e61be5 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include namespace DB @@ -59,6 +60,18 @@ void setSSLParams(nuraft::asio_service::options & asio_opts) } #endif +std::string getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + /// the most specialized path + if (config.has("keeper_server.snapshot_storage_path")) + return config.getString("keeper_server.snapshot_storage_path"); + + if (config.has("keeper_server.storage_path")) + return std::filesystem::path{config.getString("keeper_server.storage_path")} / "snapshots"; + + return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/snapshots"; +} + } KeeperServer::KeeperServer( @@ -71,7 +84,7 @@ KeeperServer::KeeperServer( , coordination_settings(coordination_settings_) , state_machine(nuraft::cs_new( responses_queue_, snapshots_queue_, - config.getString("keeper_server.snapshot_storage_path", config.getString("path", DBMS_DEFAULT_PATH) + "coordination/snapshots"), + getSnapshotsPathFromConfig(config), coordination_settings)) , state_manager(nuraft::cs_new(server_id, "keeper_server", config, coordination_settings)) , log(&Poco::Logger::get("KeeperServer")) diff --git a/src/Coordination/KeeperStateManager.cpp b/src/Coordination/KeeperStateManager.cpp index e57ae7e7c19..57a9608fce3 100644 --- a/src/Coordination/KeeperStateManager.cpp +++ b/src/Coordination/KeeperStateManager.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -9,6 +10,23 @@ namespace ErrorCodes extern const int RAFT_ERROR; } +namespace +{ + +std::string getLogsPathFromConfig(const std::string & config_prefix, const Poco::Util::AbstractConfiguration & config) +{ + /// the most specialized path + if (config.has(config_prefix + ".log_storage_path")) + return config.getString(config_prefix + ".log_storage_path"); + + if (config.has(config_prefix + ".storage_path")) + return std::filesystem::path{config.getString(config_prefix + ".storage_path")} / "logs"; + + return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/logs"; +} + +} + KeeperStateManager::KeeperStateManager(int server_id_, const std::string & host, int port, const std::string & logs_path) : my_server_id(server_id_) , my_port(port) @@ -28,7 +46,7 @@ KeeperStateManager::KeeperStateManager( : my_server_id(my_server_id_) , secure(config.getBool(config_prefix + ".raft_configuration.secure", false)) , log_store(nuraft::cs_new( - config.getString(config_prefix + ".log_storage_path", config.getString("path", DBMS_DEFAULT_PATH) + "coordination/logs"), + getLogsPathFromConfig(config_prefix, config), coordination_settings->rotate_log_storage_interval, coordination_settings->force_sync)) , cluster_config(nuraft::cs_new()) { From 4e3f103748a0fc5f2c575bc50e74dd389d20cd4c Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 17:09:05 +0300 Subject: [PATCH 025/202] Missed config file --- programs/keeper/keeper_config.xml | 81 +++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 programs/keeper/keeper_config.xml diff --git a/programs/keeper/keeper_config.xml b/programs/keeper/keeper_config.xml new file mode 100644 index 00000000000..ef218c9f2d7 --- /dev/null +++ b/programs/keeper/keeper_config.xml @@ -0,0 +1,81 @@ + + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + 1000M + 10 + + + + 4096 + + + 9181 + + + 1 + + /var/lib/clickhouse/coordination/logs + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + information + + + + + + 1 + + + localhost + 44444 + + + + + + + + + + + + + /etc/clickhouse-keeper/server.crt + /etc/clickhouse-keeper/server.key + + /etc/clickhouse-keeper/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + From 9f5f29f0aaf862fa6e760e34a60e728bfde4feee Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 17:16:45 +0300 Subject: [PATCH 026/202] Remove accident changes --- src/Core/Defines.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index fbcc6fabbf0..668a60f9be8 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -98,8 +98,6 @@ #define DBMS_DEFAULT_PATH "/var/lib/clickhouse/" -#define KEEPER_DEFAULT_PATH "/var/lib/clickhouse/coordination" - // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html /// Marks that extra information is sent to a shard. It could be any magic numbers. From da73ba04dab12e79d208a56f6c5e8ba3658fb0b5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 17:57:01 +0300 Subject: [PATCH 027/202] Add missed file --- programs/server/keeper_embedded.xml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 programs/server/keeper_embedded.xml diff --git a/programs/server/keeper_embedded.xml b/programs/server/keeper_embedded.xml new file mode 100644 index 00000000000..37edaedba80 --- /dev/null +++ b/programs/server/keeper_embedded.xml @@ -0,0 +1,21 @@ + + + trace + true + + + + 9181 + 1 + ./keeper_log + ./keeper_snapshot + + + + 1 + localhost + 44444 + + + + From dc55d197092613d034656f3b8d8339e1fdc32dfb Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 May 2021 23:00:28 +0300 Subject: [PATCH 028/202] Fix fasttest image --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 42c720a7e63..3c899b0dabb 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -73,7 +73,7 @@ function start_server --path "$FASTTEST_DATA" --user_files_path "$FASTTEST_DATA/user_files" --top_level_domains_path "$FASTTEST_DATA/top_level_domains" - --keeper_server.log_storage_path "$FASTTEST_DATA/coordination" + --keeper_server.storage_path "$FASTTEST_DATA/coordination" ) clickhouse-server "${opts[@]}" &>> "$FASTTEST_OUTPUT/server.log" & server_pid=$! From df22534dbc00176773c8d7c18086db3ad6439960 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 13 May 2021 11:42:10 +0300 Subject: [PATCH 029/202] Fix cmake --- programs/keeper/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index cd10baa2e57..211bd74ba3e 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -13,4 +13,4 @@ set (CLICKHOUSE_KEEPER_LINK clickhouse_program_add(keeper) -install (FILES clickhouse-keeper.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper RENAME keeper_config.xml) +install (FILES keeper_config.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper) From 7115045317999f0d06ee7c9d123f26e6588330e2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 13 May 2021 09:39:57 +0000 Subject: [PATCH 030/202] Tiny changes --- src/Storages/RabbitMQ/RabbitMQHandler.h | 8 -------- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 15 +++++++++------ src/Storages/RabbitMQ/StorageRabbitMQ.h | 6 +++--- tests/integration/test_storage_rabbitmq/test.py | 8 ++++---- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQHandler.h b/src/Storages/RabbitMQ/RabbitMQHandler.h index f1c7afffc5a..78ccf56dec2 100644 --- a/src/Storages/RabbitMQ/RabbitMQHandler.h +++ b/src/Storages/RabbitMQ/RabbitMQHandler.h @@ -18,14 +18,6 @@ namespace Loop } -class RabbitMQChannel : public AMQP::TcpChannel -{ -public: - RabbitMQChannel(AMQP::TcpConnection * connection) : TcpChannel(connection) {} - ~RabbitMQChannel() override { close(); } -}; - - class RabbitMQHandler : public AMQP::LibUvHandler { diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 0949c8ac802..8ec55c1b1c1 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -271,7 +271,7 @@ void StorageRabbitMQ::initRabbitMQ() return; } - RabbitMQChannel rabbit_channel(connection.get()); + AMQP::TcpChannel rabbit_channel(connection.get()); /// Main exchange -> Bridge exchange -> ( Sharding exchange ) -> Queues -> Consumers @@ -283,10 +283,11 @@ void StorageRabbitMQ::initRabbitMQ() LOG_TRACE(log, "RabbitMQ setup completed"); rabbit_is_ready = true; + rabbit_channel.close(); } -void StorageRabbitMQ::initExchange(RabbitMQChannel & rabbit_channel) +void StorageRabbitMQ::initExchange(AMQP::TcpChannel & rabbit_channel) { /// Exchange hierarchy: /// 1. Main exchange (defined with table settings - rabbitmq_exchange_name, rabbitmq_exchange_type). @@ -357,7 +358,7 @@ void StorageRabbitMQ::initExchange(RabbitMQChannel & rabbit_channel) } -void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) +void StorageRabbitMQ::bindExchange(AMQP::TcpChannel & rabbit_channel) { size_t bound_keys = 0; @@ -418,7 +419,7 @@ void StorageRabbitMQ::bindExchange(RabbitMQChannel & rabbit_channel) } -void StorageRabbitMQ::bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channel) +void StorageRabbitMQ::bindQueue(size_t queue_id, AMQP::TcpChannel & rabbit_channel) { auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) { @@ -574,7 +575,7 @@ void StorageRabbitMQ::unbindExchange() event_handler->updateLoopState(Loop::STOP); looping_task->deactivate(); - RabbitMQChannel rabbit_channel(connection.get()); + AMQP::TcpChannel rabbit_channel(connection.get()); rabbit_channel.removeExchange(bridge_exchange) .onSuccess([&]() { @@ -589,6 +590,7 @@ void StorageRabbitMQ::unbindExchange() { event_handler->iterateLoop(); } + rabbit_channel.close(); }); } @@ -720,7 +722,7 @@ void StorageRabbitMQ::cleanupRabbitMQ() const if (use_user_setup) return; - RabbitMQChannel rabbit_channel(connection.get()); + AMQP::TcpChannel rabbit_channel(connection.get()); for (const auto & queue : queues) { /// AMQP::ifunused is needed, because it is possible to share queues between multiple tables and dropping @@ -740,6 +742,7 @@ void StorageRabbitMQ::cleanupRabbitMQ() const }); } event_handler->startBlockingLoop(); + rabbit_channel.close(); /// Also there is no need to cleanup exchanges as they were created with AMQP::autodelete option. Once queues /// are removed, exchanges will also be cleaned. diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 0e47a8e1150..fd2da6bba21 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -159,9 +159,9 @@ private: void initRabbitMQ(); void cleanupRabbitMQ() const; - void initExchange(RabbitMQChannel & rabbit_channel); - void bindExchange(RabbitMQChannel & rabbit_channel); - void bindQueue(size_t queue_id, RabbitMQChannel & rabbit_channel); + void initExchange(AMQP::TcpChannel & rabbit_channel); + void bindExchange(AMQP::TcpChannel & rabbit_channel); + void bindQueue(size_t queue_id, AMQP::TcpChannel & rabbit_channel); bool restoreConnection(bool reconnecting); bool streamToViews(); diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 008f5313e22..1f14886e50f 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1980,7 +1980,7 @@ def test_rabbitmq_drop_table_properly(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'drop', rabbitmq_format = 'JSONEachRow', - rabbitmq_queue_base = 'rabbit_queue' + rabbitmq_queue_base = 'rabbit_queue_drop' ''') credentials = pika.PlainCredentials('root', 'clickhouse') @@ -1994,14 +1994,14 @@ def test_rabbitmq_drop_table_properly(rabbitmq_cluster): if result == "1\t2\n": break - exists = channel.queue_declare(queue='rabbit_queue', passive=True) + exists = channel.queue_declare(queue='rabbit_queue_drop', passive=True) assert(exists) instance.query("DROP TABLE test.rabbitmq_drop") time.sleep(30) try: - exists = channel.queue_declare(callback, queue='rabbit_queue', passive=True) + exists = channel.queue_declare(callback, queue='rabbit_queue_drop', passive=True) except Exception as e: exists = False @@ -2016,7 +2016,7 @@ def test_rabbitmq_queue_settings(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'rabbit_exchange', rabbitmq_format = 'JSONEachRow', - rabbitmq_queue_base = 'rabbit_queue', + rabbitmq_queue_base = 'rabbit_queue_settings', rabbitmq_queue_settings_list = 'x-max-length=10,x-overflow=reject-publish' ''') From ee46850112dcedbfb0d21db01525eaa39484bf56 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 13 May 2021 15:26:10 +0300 Subject: [PATCH 031/202] Better cmake for keeper --- programs/CMakeLists.txt | 48 +++++++++++++++++++ programs/keeper/CMakeLists.txt | 8 ++++ .../{server => keeper}/keeper_embedded.xml | 0 programs/server/CMakeLists.txt | 35 +------------- 4 files changed, 57 insertions(+), 34 deletions(-) rename programs/{server => keeper}/keeper_embedded.xml (100%) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 500e4794335..6fd4c2050b4 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -198,6 +198,54 @@ macro(clickhouse_program_add name) clickhouse_program_add_executable(${name}) endmacro() +# Embed default config files as a resource into the binary. +# This is needed for two purposes: +# 1. Allow to run the binary without download of any other files. +# 2. Allow to implement "sudo clickhouse install" tool. +# +# Arguments: target (server, client, keeper, etc.) and list of files +# +# Also dependency on TARGET_FILE is required, look at examples in programs/server and programs/keeper +macro(clickhouse_embed_binaries) + # TODO We actually need this on Mac, FreeBSD. + if (OS_LINUX) + + set(arguments_list "${ARGN}") + list(GET arguments_list 0 target) + + # for some reason cmake iterates loop including + math(EXPR arguments_count "${ARGC}-1") + + foreach(RESOURCE_POS RANGE 1 "${arguments_count}") + list(GET arguments_list "${RESOURCE_POS}" RESOURCE_FILE) + set(RESOURCE_OBJ ${RESOURCE_FILE}.o) + set(RESOURCE_OBJS ${RESOURCE_OBJS} ${RESOURCE_OBJ}) + + # https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake + # PPC64LE fails to do this with objcopy, use ld or lld instead + if (ARCH_PPC64LE) + add_custom_command(OUTPUT ${RESOURCE_OBJ} + COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -m elf64lppc -r -b binary -o "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}" ${RESOURCE_FILE}) + else() + add_custom_command(OUTPUT ${RESOURCE_OBJ} + COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary ${OBJCOPY_ARCH_OPTIONS} ${RESOURCE_FILE} "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}" + COMMAND ${OBJCOPY_PATH} --rename-section .data=.rodata,alloc,load,readonly,data,contents + "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}" "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}") + endif() + set_source_files_properties(${RESOURCE_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true) + endforeach() + + add_library(clickhouse_${target}_configs STATIC ${RESOURCE_OBJS}) + set_target_properties(clickhouse_${target}_configs PROPERTIES LINKER_LANGUAGE C) + + # whole-archive prevents symbols from being discarded for unknown reason + # CMake can shuffle each of target_link_libraries arguments with other + # libraries in linker command. To avoid this we hardcode whole-archive + # library into single string. + add_dependencies(clickhouse-${target}-lib clickhouse_${target}_configs) + endif () +endmacro() + add_subdirectory (server) add_subdirectory (client) diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 211bd74ba3e..e604d0e304e 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -2,6 +2,10 @@ set(CLICKHOUSE_KEEPER_SOURCES Keeper.cpp ) +if (OS_LINUX) + set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") +endif () + set (CLICKHOUSE_KEEPER_LINK PRIVATE clickhouse_common_config @@ -9,8 +13,12 @@ set (CLICKHOUSE_KEEPER_LINK clickhouse_common_zookeeper daemon dbms + + ${LINK_RESOURCE_LIB} ) clickhouse_program_add(keeper) install (FILES keeper_config.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-keeper" COMPONENT clickhouse-keeper) + +clickhouse_embed_binaries(keeper keeper_config.xml keeper_embedded.xml) diff --git a/programs/server/keeper_embedded.xml b/programs/keeper/keeper_embedded.xml similarity index 100% rename from programs/server/keeper_embedded.xml rename to programs/keeper/keeper_embedded.xml diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index bc7f2d94153..f7f76fdb450 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -31,37 +31,4 @@ clickhouse_program_add(server) install(FILES config.xml users.xml DESTINATION "${CLICKHOUSE_ETC_DIR}/clickhouse-server" COMPONENT clickhouse) -# TODO We actually need this on Mac, FreeBSD. -if (OS_LINUX) - # Embed default config files as a resource into the binary. - # This is needed for two purposes: - # 1. Allow to run the binary without download of any other files. - # 2. Allow to implement "sudo clickhouse install" tool. - - foreach(RESOURCE_FILE config.xml users.xml embedded.xml play.html keeper_embedded.xml) - set(RESOURCE_OBJ ${RESOURCE_FILE}.o) - set(RESOURCE_OBJS ${RESOURCE_OBJS} ${RESOURCE_OBJ}) - - # https://stackoverflow.com/questions/14776463/compile-and-add-an-object-file-from-a-binary-with-cmake - # PPC64LE fails to do this with objcopy, use ld or lld instead - if (ARCH_PPC64LE) - add_custom_command(OUTPUT ${RESOURCE_OBJ} - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -m elf64lppc -r -b binary -o "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}" ${RESOURCE_FILE}) - else() - add_custom_command(OUTPUT ${RESOURCE_OBJ} - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${OBJCOPY_PATH} -I binary ${OBJCOPY_ARCH_OPTIONS} ${RESOURCE_FILE} "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}" - COMMAND ${OBJCOPY_PATH} --rename-section .data=.rodata,alloc,load,readonly,data,contents - "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}" "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}") - endif() - set_source_files_properties(${RESOURCE_OBJ} PROPERTIES EXTERNAL_OBJECT true GENERATED true) - endforeach(RESOURCE_FILE) - - add_library(clickhouse_server_configs STATIC ${RESOURCE_OBJS}) - set_target_properties(clickhouse_server_configs PROPERTIES LINKER_LANGUAGE C) - - # whole-archive prevents symbols from being discarded for unknown reason - # CMake can shuffle each of target_link_libraries arguments with other - # libraries in linker command. To avoid this we hardcode whole-archive - # library into single string. - add_dependencies(clickhouse-server-lib clickhouse_server_configs) -endif () +clickhouse_embed_binaries(server config.xml users.xml embedded.xml play.html) From dc6eb48197fb5b9bbb538c5f047777f1e5c62850 Mon Sep 17 00:00:00 2001 From: Evgeniia Sudarikova Date: Thu, 13 May 2021 18:05:48 +0300 Subject: [PATCH 032/202] edited after review --- .../functions/type-conversion-functions.md | 9 +++++--- .../functions/type-conversion-functions.md | 22 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 87dbfa1728f..8a477b79836 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -469,7 +469,8 @@ Converts input value `x` to the `T` data type. Unlike to `reinterpret` function, The syntax `CAST(x AS t)` is also supported. -Note, that if value `x` does not fit the bounds of type `T`, the function overflows. For example, `CAST(-1, 'UInt8')` returns `255`. +!!! note "Note" + If value `x` does not fit the bounds of type `T`, the function overflows. For example, `CAST(-1, 'UInt8')` returns `255`. **Syntax** @@ -1167,8 +1168,10 @@ Result: ## toUnixTimestamp64Nano {#tounixtimestamp64nano} -Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. -Input value is scaled up or down appropriately depending on it precision. Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`. +Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision. + +!!! note "Note" + Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`. **Syntax** diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 46032f29551..d86de71ec72 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -369,7 +369,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut; ## reinterpretAsUUID {#reinterpretasuuid} -Функция принимает шестнадцатибайтную строку и интерпретирует ее байты в network order (big-endian). Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количеством нулевых байтов с конца. Если строка длиннее, чем шестнадцать байтов, то игнорируются лишние байты с конца. +Функция принимает строку из 16 байт и интерпретирует ее байты в порядок байт от старшего к младшему. Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количеством нулевых байтов с конца. Если строка длиннее, чем 16 байтов, то лишние байты с конца игнорируются. **Синтаксис** @@ -466,7 +466,8 @@ SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint, Поддерживается также синтаксис `CAST(x AS t)`. -Обратите внимание, что если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255. +!!! note "Примечание" + Обратите внимание, что если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255. **Синтаксис** @@ -479,7 +480,7 @@ CAST(x, T) - `x` — любой тип данных. - `T` — конечный тип данных. [String](../../sql-reference/data-types/string.md). -**Returned value** +**Возвращаемое значение** - Конечный тип данных. @@ -523,7 +524,7 @@ SELECT Преобразование в FixedString(N) работает только для аргументов типа [String](../../sql-reference/data-types/string.md) или [FixedString](../../sql-reference/data-types/fixedstring.md). -Поддержано преобразование к типу [Nullable](../../sql-reference/functions/type-conversion-functions.md) и обратно. +Поддерживается преобразование к типу [Nullable](../../sql-reference/functions/type-conversion-functions.md) и обратно. **Примеры** @@ -893,7 +894,7 @@ AS parseDateTimeBestEffortUS; ## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero} ## parseDateTime32BestEffortOrZero {#parsedatetime32besteffortorzero} -Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевую дату или нулевую дату и время, когда получает формат даты, который не может быть обработан. +Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевую дату или нулевую дату и время в случае, если получает формат даты, который не может быть обработан. ## parseDateTimeBestEffortUSOrNull {#parsedatetimebesteffortusornull} @@ -1077,7 +1078,7 @@ SELECT parseDateTimeBestEffortUSOrZero('02.2021') AS parseDateTimeBestEffortUSOr parseDateTime64BestEffort(time_string [, precision [, time_zone]]) ``` -**Параметры** +**Аргументы** - `time_string` — строка, содержащая дату или дату со временем, которые нужно преобразовать. [String](../../sql-reference/data-types/string.md). - `precision` — `3` для миллисекунд, `6` для микросекунд. По умолчанию `3`. Необязательный. [UInt8](../../sql-reference/data-types/int-uint.md). @@ -1115,11 +1116,11 @@ FORMAT PrettyCompactMonoBlcok; ## parseDateTime64BestEffortOrNull {#parsedatetime32besteffortornull} -Работает аналогично функции [parseDateTime64BestEffort](#parsedatetime64besteffort), но возвращает `NULL`, когда встречает формат даты, который не может обработать. +Работает аналогично функции [parseDateTime64BestEffort](#parsedatetime64besteffort), но возвращает `NULL` в случае, если встречает формат даты, который не может обработать. ## parseDateTime64BestEffortOrZero {#parsedatetime64besteffortorzero} -Работает аналогично функции [parseDateTime64BestEffort](#parsedatetimebesteffort), но возвращает "нулевую" дату и время, когда встречает формат даты, который не может обработать. +Работает аналогично функции [parseDateTime64BestEffort](#parsedatetimebesteffort), но возвращает нулевую дату и время в случае, если встречает формат даты, который не может обработать. ## toLowCardinality {#tolowcardinality} @@ -1166,7 +1167,10 @@ SELECT toLowCardinality('1'); ## toUnixTimestamp64Nano {#tounixtimestamp64nano} Преобразует значение `DateTime64` в значение `Int64` с фиксированной точностью менее одной секунды. -Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. Обратите внимание, что возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`. +Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. + +!!! note "Примечание" + Обратите внимание, что возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`. **Синтаксис** From 9744925aa99991af7ac6b4b873a75a58a07f1a8b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 13 May 2021 19:33:07 +0300 Subject: [PATCH 033/202] done --- CHANGELOG.md | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc1ec835a7b..6a2c7cb66b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,148 @@ +## ClickHouse release 21.5 + +#### Backward Incompatible Change + +* Change comparison of integers and floating point numbers when integer is not exactly representable in the floating point data type. In new version comparison will return false as the rounding error will occur. Example: `9223372036854775808.0 != 9223372036854775808`, because the number `9223372036854775808` is not representable as floating point number exactly (and `9223372036854775808.0` is rounded to `9223372036854776000.0`). But in previous version the comparison will return as the numbers are equal, because if the floating point number `9223372036854776000.0` get converted back to UInt64, it will yield `9223372036854775808`. For the reference, the Python programming language also treats these numbers as equal. But this behaviour was dependend on CPU model (different results on AMD64 and AArch64 for some out-of-range numbers), so we make the comparison more precise. It will treat int and float numbers equal only if int is represented in floating point type exactly. [#22595](https://github.com/ClickHouse/ClickHouse/pull/22595) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)). + +#### New Feature + +* Added `Decimal256` type support in dictionaries. Closes [#20979](https://github.com/ClickHouse/ClickHouse/issues/20979). [#22960](https://github.com/ClickHouse/ClickHouse/pull/22960) ([Maksim Kita](https://github.com/kitaisreal)). +* Added `ALTER TABLE ... FETCH PART ...` query. It's similar to `FETCH PARTITION`, but fetches only one part. [#22706](https://github.com/ClickHouse/ClickHouse/pull/22706) ([turbo jason](https://github.com/songenjie)). +* FlatDictionary added `initial_array_size`, `max_array_size` options. [#22521](https://github.com/ClickHouse/ClickHouse/pull/22521) ([Maksim Kita](https://github.com/kitaisreal)). +* Add new setting `non_replicated_deduplication_window` for non-replicated MergeTree inserts deduplication. [#22514](https://github.com/ClickHouse/ClickHouse/pull/22514) ([alesapin](https://github.com/alesapin)). +* Update paths to the `CatBoost` model configs in config reloading. [#22434](https://github.com/ClickHouse/ClickHouse/pull/22434) ([Kruglov Pavel](https://github.com/Avogar)). +* Added function `dictGetOrNull`. It works like `dictGet`, but return `Null` in case key was not found in dictionary. Closes [#22375](https://github.com/ClickHouse/ClickHouse/issues/22375). [#22413](https://github.com/ClickHouse/ClickHouse/pull/22413) ([Maksim Kita](https://github.com/kitaisreal)). +* Added support for replicas and shards in MySQL/PostgreSQL table engine / table function. Replicas and shards, if used together, are added via new table engine ExternalDistributed. Closes [#20969](https://github.com/ClickHouse/ClickHouse/issues/20969). [#22217](https://github.com/ClickHouse/ClickHouse/pull/22217) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improved performance of `dictGetHierarchy`, `dictIsIn` functions. Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)). +* Added a table function `s3Cluster`, which allows to process files from `s3` in parallel on every node of a specified cluster. [#22012](https://github.com/ClickHouse/ClickHouse/pull/22012) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Added a setting `max_distributed_depth` that limits the depth of recursive queries to `Distributed` tables. Closes [#20229](https://github.com/ClickHouse/ClickHouse/issues/20229). [#21942](https://github.com/ClickHouse/ClickHouse/pull/21942) ([flynn](https://github.com/ucasFL)). +* Added function `arrayFold(x1,...,xn,accum -> expression, array1,...,arrayn, init_accum)` that applies the expression to each element of the array (or set of parallel arrays) and collect result in accumulator. [#21589](https://github.com/ClickHouse/ClickHouse/pull/21589) ([Dmitry Krylov](https://github.com/dmalkr)). + +#### Performance Improvement + +* Improved performance of `intDiv` by dynamic dispatch for AVX2. This closes [#22314](https://github.com/ClickHouse/ClickHouse/issues/22314). [#23000](https://github.com/ClickHouse/ClickHouse/pull/23000) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improved performance of reading from `ArrowStream` input format for sources other then local file (e.g. URL). [#22673](https://github.com/ClickHouse/ClickHouse/pull/22673) ([nvartolomei](https://github.com/nvartolomei)). +* Disabled compression by default when interacting with localhost (with clickhouse-client or server to server with distributed queries) via native protocol. It may improve performance of some import/export operations. This closes [#22234](https://github.com/ClickHouse/ClickHouse/issues/22234). [#22237](https://github.com/ClickHouse/ClickHouse/pull/22237) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Exclude values that does not belong to the shard from right part of IN section for distributed queries. [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)). +* Improved performance of reading a subset of columns with File-like table engine and column-oriented format like Parquet, Arrow or ORC. This closes [#issue:20129](https://github.com/ClickHouse/ClickHouse/issues/20129). [#21302](https://github.com/ClickHouse/ClickHouse/pull/21302) ([keenwolf](https://github.com/keen-wolf)). + +#### Improvement + +* Enabled `async_socket_for_remote` by default. [#23683](https://github.com/ClickHouse/ClickHouse/pull/23683) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed `quantile(s)TDigest`. Added special handling of singleton centroids according to tdunning/t-digest 3.2+. Also a bug with over-compression of centroids in implementation of earlier version of the algorithm was fixed. [#23314](https://github.com/ClickHouse/ClickHouse/pull/23314) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Disable settings `use_hedged_requests` and `async_socket_for_remote` because there is an evidence that it may cause issues. [#23261](https://github.com/ClickHouse/ClickHouse/pull/23261) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Make function `unhex` case insensitive for compatibility with MySQL. [#23229](https://github.com/ClickHouse/ClickHouse/pull/23229) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed memory tracking with min_bytes_to_use_mmap_io. [#23211](https://github.com/ClickHouse/ClickHouse/pull/23211) ([Azat Khuzhin](https://github.com/azat)). +* Implement functions `arrayHasAny`, `arrayHasAll`, `has`, `indexOf`, `countEqual` for generic case when types of array elements are different. In previous versions the functions `arrayHasAny`, `arrayHasAll` returned false and `has`, `indexOf`, `countEqual` thrown exception. Also add support for `Decimal` and big integer types in functions `has` and similar. This closes [#20272](https://github.com/ClickHouse/ClickHouse/issues/20272). [#23044](https://github.com/ClickHouse/ClickHouse/pull/23044) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Raised the threshold on max number of matches in result of the function `extractAllGroupsHorizontal`. [#23036](https://github.com/ClickHouse/ClickHouse/pull/23036) ([Vasily Nemkov](https://github.com/Enmk)). +* Do not perform `optimize_skip_unused_shards` for cluster with one node. [#22999](https://github.com/ClickHouse/ClickHouse/pull/22999) ([Azat Khuzhin](https://github.com/azat)). +* Increase `max_uri_size` (the maximum size of URL in HTTP interface) to 1 MiB by default. This closes [#21197](https://github.com/ClickHouse/ClickHouse/issues/21197). [#22997](https://github.com/ClickHouse/ClickHouse/pull/22997) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Added ability to run clickhouse-keeper with SSL. Config settings `keeper_server.tcp_port_secure` can be used for secure interaction between client and keeper-server. `keeper_server.raft_configuration.secure` can be used to enable internal secure communication between nodes. [#22992](https://github.com/ClickHouse/ClickHouse/pull/22992) ([alesapin](https://github.com/alesapin)). +* Added ability to flush buffer only in background for StorageBuffer. [#22986](https://github.com/ClickHouse/ClickHouse/pull/22986) ([Azat Khuzhin](https://github.com/azat)). +* When selecting from MergeTree table with NULL in WHERE condition, in rare cases, exception was thrown. This closes [#20019](https://github.com/ClickHouse/ClickHouse/issues/20019). [#22978](https://github.com/ClickHouse/ClickHouse/pull/22978) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed an error handling in Poco HTTP Client for AWS. [#22973](https://github.com/ClickHouse/ClickHouse/pull/22973) ([kreuzerkrieg](https://github.com/kreuzerkrieg)). +* Respect `max_part_removal_threads` for ReplicatedMergeTree. [#22971](https://github.com/ClickHouse/ClickHouse/pull/22971) ([Azat Khuzhin](https://github.com/azat)). +* Fix inactive_parts_to_throw_insert=0 with inactive_parts_to_delay_insert>0. [#22947](https://github.com/ClickHouse/ClickHouse/pull/22947) ([Azat Khuzhin](https://github.com/azat)). +* Set `background_fetches_pool_size` to 8 that is better for production usage with frequent small insertions or slow ZooKeeper cluster. [#22945](https://github.com/ClickHouse/ClickHouse/pull/22945) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* `dateDiff` now works with `DateTime64` arguments (even for values outside of `DateTime` range) [#22931](https://github.com/ClickHouse/ClickHouse/pull/22931) ([Vasily Nemkov](https://github.com/Enmk)). +* Added an ability to replicate MySQL databases containing views without failing. This is accomplished by ignoring the views. [#22760](https://github.com/ClickHouse/ClickHouse/pull/22760) ([Christian](https://github.com/cfroystad)). +* Allow RBAC row policy via postgresql protocol. Closes [#22658](https://github.com/ClickHouse/ClickHouse/issues/22658). PostgreSQL protocol is enabled in configuration by default. [#22755](https://github.com/ClickHouse/ClickHouse/pull/22755) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add metric to track how much time is spend during waiting for Buffer layer lock. [#22725](https://github.com/ClickHouse/ClickHouse/pull/22725) ([Azat Khuzhin](https://github.com/azat)). +* Allow to use CTE in VIEW definition. This closes [#22491](https://github.com/ClickHouse/ClickHouse/issues/22491). [#22657](https://github.com/ClickHouse/ClickHouse/pull/22657) ([Amos Bird](https://github.com/amosbird)). +* Clear the rest of the screen and show cursor in `clickhouse-client` if previous program has left garbage in terminal. This closes [#16518](https://github.com/ClickHouse/ClickHouse/issues/16518). [#22634](https://github.com/ClickHouse/ClickHouse/pull/22634) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Make `round` function to behave consistently on non-x86_64 platforms. Rounding half to nearest even (Banker's rounding) is used. [#22582](https://github.com/ClickHouse/ClickHouse/pull/22582) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Correctly check structure of async distributed blocks. [#22325](https://github.com/ClickHouse/ClickHouse/pull/22325) ([Azat Khuzhin](https://github.com/azat)). +* Added nanodbc instead of Poco::ODBC. Closes [#9678](https://github.com/ClickHouse/ClickHouse/issues/9678). Add support for DateTime64 and Decimal* for ODBC table engine. Closes [#21961](https://github.com/ClickHouse/ClickHouse/issues/21961). Fixed issue with cyrillic text being truncated. Closes [#16246](https://github.com/ClickHouse/ClickHouse/issues/16246). Added connection pools for odbc bridge. [#21972](https://github.com/ClickHouse/ClickHouse/pull/21972) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Allow publishing Kafka errors to a virtual column of Kafka engine, controlled by the `kafka_handle_error_mode` setting. [#21850](https://github.com/ClickHouse/ClickHouse/pull/21850) ([fastio](https://github.com/fastio)). +* Add aliases `simpleJSONExtract/simpleJSONHas` to `visitParam/visitParamExtract{UInt, Int, Bool, Float, Raw, String}`. Fixes [#21383](https://github.com/ClickHouse/ClickHouse/issues/21383). [#21519](https://github.com/ClickHouse/ClickHouse/pull/21519) ([fastio](https://github.com/fastio)). +* Add `clickhouse-library-bridge` for library dictionary source. Closes [#9502](https://github.com/ClickHouse/ClickHouse/issues/9502). [#21509](https://github.com/ClickHouse/ClickHouse/pull/21509) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasFL)). +* Support dynamic interserver credentials. [#14113](https://github.com/ClickHouse/ClickHouse/pull/14113) ([johnskopis](https://github.com/johnskopis)). + +#### Bug Fix + +* HashedDictionary complex key update field initial load fix. Closes [#23800](https://github.com/ClickHouse/ClickHouse/issues/23800). [#23824](https://github.com/ClickHouse/ClickHouse/pull/23824) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed an error `Can't initialize pipeline with empty pipe` for queries with `GLOBAL IN/JOIN` and `use_hedged_requests`. Fixes [#23431](https://github.com/ClickHouse/ClickHouse/issues/23431). [#23805](https://github.com/ClickHouse/ClickHouse/pull/23805) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed remote JDBC bridge timeout connection issue. Closes [#9609](https://github.com/ClickHouse/ClickHouse/issues/9609). [#23771](https://github.com/ClickHouse/ClickHouse/pull/23771) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed crash when `PREWHERE` and row policy filter are both in effect with empty result. [#23763](https://github.com/ClickHouse/ClickHouse/pull/23763) ([Amos Bird](https://github.com/amosbird)). +* Avoid possible "Cannot schedule a task" error (in case some exception had been occurred) on INSERT into Distributed. [#23744](https://github.com/ClickHouse/ClickHouse/pull/23744) ([Azat Khuzhin](https://github.com/azat)). +* Added an exception in case of completely the same values in both samples in aggregate function `mannWhitneyUTest`. This fixes [#23646](https://github.com/ClickHouse/ClickHouse/issues/23646). [#23654](https://github.com/ClickHouse/ClickHouse/pull/23654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed server fault when inserting data through HTTP caused an exception. This fixes [#23512](https://github.com/ClickHouse/ClickHouse/issues/23512). [#23643](https://github.com/ClickHouse/ClickHouse/pull/23643) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed misinterpretation of some `LIKE` expressions with escape sequences. [#23610](https://github.com/ClickHouse/ClickHouse/pull/23610) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed restart / stop command hanging. Closes [#20214](https://github.com/ClickHouse/ClickHouse/issues/20214). [#23552](https://github.com/ClickHouse/ClickHouse/pull/23552) ([filimonov](https://github.com/filimonov)). +* Fixed `columns` function when multiple joins in select query. Closes [#22736](https://github.com/ClickHouse/ClickHouse/issues/22736). [#23501](https://github.com/ClickHouse/ClickHouse/pull/23501) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed a crash when modifying column's default value when a column itself is used as ReplacingMergeTree's parameter. [#23483](https://github.com/ClickHouse/ClickHouse/pull/23483) ([hexiaoting](https://github.com/hexiaoting)). +* Fixed corner cases in vertical merges with `ReplacingMergeTree`. In rare cases they could lead to fails of merges with exceptions like `Incomplete granules are not allowed while blocks are granules size`. [#23459](https://github.com/ClickHouse/ClickHouse/pull/23459) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed bug that does not allow cast from empty array literal, to array with dimensions greater than 1. Closes [#14476](https://github.com/ClickHouse/ClickHouse/issues/14476). [#23456](https://github.com/ClickHouse/ClickHouse/pull/23456) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed a bug when `deltaSum` aggregate function produced incorrect result after resetting the counter. [#23437](https://github.com/ClickHouse/ClickHouse/pull/23437) ([Russ Frank](https://github.com/rf)). +* Fixed `Cannot unlink file` error on unsuccessful creation of ReplicatedMergeTree table with multidisk configuration. This closes [#21755](https://github.com/ClickHouse/ClickHouse/issues/21755). [#23433](https://github.com/ClickHouse/ClickHouse/pull/23433) ([tavplubix](https://github.com/tavplubix)). +* Fixed a lack of support for Kafka storage with `arrow` and `arrowstream` format messages. [#23415](https://github.com/ClickHouse/ClickHouse/pull/23415) ([Chao Ma](https://github.com/godliness)). +* Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)). +* Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)). +* Fixed a bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)). +* Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)). +* Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)). +* Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([tavplubix](https://github.com/tavplubix)). +* Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)). +* Add type conversion for `optimize_skip_unused_shards_rewrite_in` (fixes `use-of-uninitialized-value` with `optimize_skip_unused_shards_rewrite_in`). [#23219](https://github.com/ClickHouse/ClickHouse/pull/23219) ([Azat Khuzhin](https://github.com/azat)). +* Fixed reading from ODBC when there are many long column names in a table. Closes [#8853](https://github.com/ClickHouse/ClickHouse/issues/8853). [#23215](https://github.com/ClickHouse/ClickHouse/pull/23215) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed the behavior when disabling `input_format_with_names_use_header ` setting discards all the input with CSVWithNames format. This fixes [#22406](https://github.com/ClickHouse/ClickHouse/issues/22406). [#23202](https://github.com/ClickHouse/ClickHouse/pull/23202) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed `Not found column` error when selecting from `MaterializeMySQL` with condition on key column. Fixes [#22432](https://github.com/ClickHouse/ClickHouse/issues/22432). [#23200](https://github.com/ClickHouse/ClickHouse/pull/23200) ([tavplubix](https://github.com/tavplubix)). +* QueryAliasVisitor to prefer alias for ASTWithAlias if subquery was optimized to constant. Fixes [#22924](https://github.com/ClickHouse/ClickHouse/issues/22924). Fixes [#10401](https://github.com/ClickHouse/ClickHouse/issues/10401). [#23191](https://github.com/ClickHouse/ClickHouse/pull/23191) ([Maksim Kita](https://github.com/kitaisreal)). +* Server might fail to start if `data_type_default_nullable` setting is enabled in default profile, it's fixed. Fixes [#22573](https://github.com/ClickHouse/ClickHouse/issues/22573). [#23185](https://github.com/ClickHouse/ClickHouse/pull/23185) ([tavplubix](https://github.com/tavplubix)). +* Fixed a crash on shutdown which happened because of wrong accounting of current connections. [#23154](https://github.com/ClickHouse/ClickHouse/pull/23154) ([Vitaly Baranov](https://github.com/vitlibar)). +* Some values were formatted with alignment in center in table cells in `Markdown` format. Not anymore. [#23096](https://github.com/ClickHouse/ClickHouse/pull/23096) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fixed `Table .inner_id... doesn't exist` error when selecting from Materialized View after detaching it from Atomic database and attaching back. [#23047](https://github.com/ClickHouse/ClickHouse/pull/23047) ([tavplubix](https://github.com/tavplubix)). +* Remove non-essential details from suggestions in clickhouse-client. This closes [#22158](https://github.com/ClickHouse/ClickHouse/issues/22158). [#23040](https://github.com/ClickHouse/ClickHouse/pull/23040) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix error `Cannot find column in ActionsDAG result` which may happen if subquery uses `untuple`. Fixes [#22290](https://github.com/ClickHouse/ClickHouse/issues/22290). [#22991](https://github.com/ClickHouse/ClickHouse/pull/22991) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix usage of constant columns of type `Map` with nullable values. [#22939](https://github.com/ClickHouse/ClickHouse/pull/22939) ([Anton Popov](https://github.com/CurtizJ)). +* fixed `formatDateTime()` on `DateTime64` and "%C" format specifier fixed `toDateTime64()` for large values and non-zero scale. ... [#22937](https://github.com/ClickHouse/ClickHouse/pull/22937) ([Vasily Nemkov](https://github.com/Enmk)). +* Fixed a crash when using `mannWhitneyUTest` and `rankCorr` with window functions. This fixes [#22728](https://github.com/ClickHouse/ClickHouse/issues/22728). [#22876](https://github.com/ClickHouse/ClickHouse/pull/22876) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fixed `bytes_allocated` field in system.dictionaries for sparse_hashed dictionaries. [#22867](https://github.com/ClickHouse/ClickHouse/pull/22867) ([Azat Khuzhin](https://github.com/azat)). +* Fixed possible hanging in concurrent DROP/CREATE of TEMPORARY LIVE VIEW in `TemporaryLiveViewCleaner`, see https://gist.github.com/vzakaznikov/0c03195960fc86b56bfe2bc73a90019e. [#22858](https://github.com/ClickHouse/ClickHouse/pull/22858) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed pushdown of `HAVING` in case, when filter column is used in aggregation. [#22763](https://github.com/ClickHouse/ClickHouse/pull/22763) ([Anton Popov](https://github.com/CurtizJ)). +* Fixed approximate total rows accounting for reverse reading from MergeTree. [#22726](https://github.com/ClickHouse/ClickHouse/pull/22726) ([Azat Khuzhin](https://github.com/azat)). +* Fixed possible hangs in Zookeeper requests in case of OOM exception. Fixes [#22438](https://github.com/ClickHouse/ClickHouse/issues/22438). [#22684](https://github.com/ClickHouse/ClickHouse/pull/22684) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed wait for mutations on several replicas for ReplicatedMergeTree table engines. Previously, mutation/alter query may finish before mutation actually executed on other replicas. [#22669](https://github.com/ClickHouse/ClickHouse/pull/22669) ([alesapin](https://github.com/alesapin)). +* Fixed LOGICAL_ERROR for Log with nested types w/o columns in the SELECT clause. [#22654](https://github.com/ClickHouse/ClickHouse/pull/22654) ([Azat Khuzhin](https://github.com/azat)). +* Fixed a bug with unlimited wait for auxiliary AWS requests. [#22594](https://github.com/ClickHouse/ClickHouse/pull/22594) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Fixed a crash when client closes connection very early [#22579](https://github.com/ClickHouse/ClickHouse/issues/22579). [#22591](https://github.com/ClickHouse/ClickHouse/pull/22591) ([nvartolomei](https://github.com/nvartolomei)). +* Fixed an incorrect formatting of function `map` in distributed queries. [#22588](https://github.com/ClickHouse/ClickHouse/pull/22588) ([foolchi](https://github.com/foolchi)). +* Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). +* Fixed UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). +* Fixed deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)). +* Fixed `ClickHouseDictionarySource` configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)). +* Fixed a race condition in HedgedConnections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)). +* Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Do not limit HTTP chunk size. Fixes [#21907](https://github.com/ClickHouse/ClickHouse/issues/21907). [#22322](https://github.com/ClickHouse/ClickHouse/pull/22322) ([Ivan](https://github.com/abyss7)). +* Fixed a bug, which leads to underaggregation of data in case of enabled `optimize_aggregation_in_order` and many parts in table. Slightly improve performance of aggregation with enabled `optimize_aggregation_in_order`. [#21889](https://github.com/ClickHouse/ClickHouse/pull/21889) ([Anton Popov](https://github.com/CurtizJ)). +* Check if table function view is used as a column. This complements https://github.com/ClickHouse/ClickHouse/pull/20350. [#21465](https://github.com/ClickHouse/ClickHouse/pull/21465) ([Amos Bird](https://github.com/amosbird)). +* * Fix "unknown column" error for tables with `Merge` engine in queris with `JOIN` and aggregation. Closes [#18368](https://github.com/ClickHouse/ClickHouse/issues/18368), close [#22226](https://github.com/ClickHouse/ClickHouse/issues/22226). [#21370](https://github.com/ClickHouse/ClickHouse/pull/21370) ([Vladimir](https://github.com/vdimir)). +* Fixed name clashes in `PredicateRewriteVisitor`. It caused incorrect `WHERE` filtration after full join. Close [#20497](https://github.com/ClickHouse/ClickHouse/issues/20497). [#20622](https://github.com/ClickHouse/ClickHouse/pull/20622) ([Vladimir](https://github.com/vdimir)). +* Fixed very rare bug when quorum insert with `quorum_parallel=1` is not really "quorum" because of deduplication. [#18215](https://github.com/ClickHouse/ClickHouse/pull/18215) ([filimonov](https://github.com/filimonov)). + +#### Build/Testing/Packaging Improvement + +* Simplify debian packages. This fixes [#21698](https://github.com/ClickHouse/ClickHouse/issues/21698). [#22976](https://github.com/ClickHouse/ClickHouse/pull/22976) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Added support for ClickHouse build on Apple M1. [#21639](https://github.com/ClickHouse/ClickHouse/pull/21639) ([changvvb](https://github.com/changvvb)). +* Fixed ClickHouse Keeper build for MacOS. [#22860](https://github.com/ClickHouse/ClickHouse/pull/22860) ([alesapin](https://github.com/alesapin)). +* Fixed some tests on AArch64 platform. [#22596](https://github.com/ClickHouse/ClickHouse/pull/22596) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Added function alignment for possibly better performance. [#21431](https://github.com/ClickHouse/ClickHouse/pull/21431) ([Danila Kutenin](https://github.com/danlark1)). +* Adjust some tests to output identical results on amd64 and aarch64 (qemu). The result was depending on implementation specific CPU behaviour. [#22590](https://github.com/ClickHouse/ClickHouse/pull/22590) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow query profiling only on x86_64. See [#15174](https://github.com/ClickHouse/ClickHouse/issues/15174)#issuecomment-812954965 and [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638)#issuecomment-703805337. This closes [#15638](https://github.com/ClickHouse/ClickHouse/issues/15638). [#22580](https://github.com/ClickHouse/ClickHouse/pull/22580) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow building with unbundled xz (lzma) using USE_INTERNAL_XZ_LIBRARY=OFF ... [#22571](https://github.com/ClickHouse/ClickHouse/pull/22571) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Enable the bundled openldap on ppc64le ... [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Disable incompatible libraries (platform specific typically) on ppc64le ... [#22475](https://github.com/ClickHouse/ClickHouse/pull/22475) ([Kfir Itzhak](https://github.com/mastertheknife)). +* Add on-demand check for clickhouse Keeper. [#22373](https://github.com/ClickHouse/ClickHouse/pull/22373) ([alesapin](https://github.com/alesapin)). +* Run stateless tests in parallel in CI. Depends on [#22181](https://github.com/ClickHouse/ClickHouse/issues/22181). [#22300](https://github.com/ClickHouse/ClickHouse/pull/22300) ([alesapin](https://github.com/alesapin)). +* Build `jemalloc` with support for [heap profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling). [#22834](https://github.com/ClickHouse/ClickHouse/pull/22834) ([nvartolomei](https://github.com/nvartolomei)). + ## ClickHouse release 21.4 ### ClickHouse release 21.4.1 2021-04-12 From 1330ef7567423b3e063b6062218df1b39cb96dfd Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 13 May 2021 20:04:44 +0300 Subject: [PATCH 034/202] Fixed converting const DateTime to DateTime64 in WHERE --- src/Interpreters/convertFieldToType.cpp | 8 +++- ...866_datetime64_cmp_with_constant.reference | 12 ++++++ .../01866_datetime64_cmp_with_constant.sql | 40 +++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01866_datetime64_cmp_with_constant.reference create mode 100644 tests/queries/0_stateless/01866_datetime64_cmp_with_constant.sql diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index fa49b730379..0b124634fec 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -201,7 +201,13 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID return src; } - /// TODO Conversion from integers to DateTime64 + if (which_type.isDateTime64() + && (which_from_type.isNativeInt() || which_from_type.isNativeUInt() || which_from_type.isDateOrDateTime())) + { + const auto scale = static_cast(type).getScale(); + const auto decimal_value = DecimalUtils::decimalFromComponents(src.reinterpret(), 0, scale); + return Field(DecimalField(decimal_value, scale)); + } } else if (which_type.isUUID() && src.getType() == Field::Types::UUID) { diff --git a/tests/queries/0_stateless/01866_datetime64_cmp_with_constant.reference b/tests/queries/0_stateless/01866_datetime64_cmp_with_constant.reference new file mode 100644 index 00000000000..db516fa83d4 --- /dev/null +++ b/tests/queries/0_stateless/01866_datetime64_cmp_with_constant.reference @@ -0,0 +1,12 @@ +dt64 <= const dt +dt64 <= dt +dt <= const dt64 +dt <= dt64 +dt64 = const dt +dt64 = dt +dt = const dt64 +dt = dt64 +dt64 >= const dt +dt64 >= dt +dt >= const dt64 +dt >= dt64 diff --git a/tests/queries/0_stateless/01866_datetime64_cmp_with_constant.sql b/tests/queries/0_stateless/01866_datetime64_cmp_with_constant.sql new file mode 100644 index 00000000000..e6782656887 --- /dev/null +++ b/tests/queries/0_stateless/01866_datetime64_cmp_with_constant.sql @@ -0,0 +1,40 @@ +CREATE TABLE dt64test +( + `dt64_column` DateTime64(3), + `dt_column` DateTime DEFAULT toDateTime(dt64_column) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(dt64_column) +ORDER BY dt64_column; + +INSERT INTO dt64test (`dt64_column`) VALUES ('2020-01-13 13:37:00'); + +SELECT 'dt64 < const dt' FROM dt64test WHERE dt64_column < toDateTime('2020-01-13 13:37:00'); +SELECT 'dt64 < dt' FROM dt64test WHERE dt64_column < materialize(toDateTime('2020-01-13 13:37:00')); +SELECT 'dt < const dt64' FROM dt64test WHERE dt_column < toDateTime64('2020-01-13 13:37:00', 3); +SELECT 'dt < dt64' FROM dt64test WHERE dt_column < materialize(toDateTime64('2020-01-13 13:37:00', 3)); + +SELECT 'dt64 <= const dt' FROM dt64test WHERE dt64_column <= toDateTime('2020-01-13 13:37:00'); +SELECT 'dt64 <= dt' FROM dt64test WHERE dt64_column <= materialize(toDateTime('2020-01-13 13:37:00')); +SELECT 'dt <= const dt64' FROM dt64test WHERE dt_column <= toDateTime64('2020-01-13 13:37:00', 3); +SELECT 'dt <= dt64' FROM dt64test WHERE dt_column <= materialize(toDateTime64('2020-01-13 13:37:00', 3)); + +SELECT 'dt64 = const dt' FROM dt64test WHERE dt64_column = toDateTime('2020-01-13 13:37:00'); +SELECT 'dt64 = dt' FROM dt64test WHERE dt64_column = materialize(toDateTime('2020-01-13 13:37:00')); +SELECT 'dt = const dt64' FROM dt64test WHERE dt_column = toDateTime64('2020-01-13 13:37:00', 3); +SELECT 'dt = dt64' FROM dt64test WHERE dt_column = materialize(toDateTime64('2020-01-13 13:37:00', 3)); + +SELECT 'dt64 >= const dt' FROM dt64test WHERE dt64_column >= toDateTime('2020-01-13 13:37:00'); +SELECT 'dt64 >= dt' FROM dt64test WHERE dt64_column >= materialize(toDateTime('2020-01-13 13:37:00')); +SELECT 'dt >= const dt64' FROM dt64test WHERE dt_column >= toDateTime64('2020-01-13 13:37:00', 3); +SELECT 'dt >= dt64' FROM dt64test WHERE dt_column >= materialize(toDateTime64('2020-01-13 13:37:00', 3)); + +SELECT 'dt64 > const dt' FROM dt64test WHERE dt64_column > toDateTime('2020-01-13 13:37:00'); +SELECT 'dt64 > dt' FROM dt64test WHERE dt64_column > materialize(toDateTime('2020-01-13 13:37:00')); +SELECT 'dt > const dt64' FROM dt64test WHERE dt_column > toDateTime64('2020-01-13 13:37:00', 3); +SELECT 'dt > dt64' FROM dt64test WHERE dt_column > materialize(toDateTime64('2020-01-13 13:37:00', 3)); + +SELECT 'dt64 != const dt' FROM dt64test WHERE dt64_column != toDateTime('2020-01-13 13:37:00'); +SELECT 'dt64 != dt' FROM dt64test WHERE dt64_column != materialize(toDateTime('2020-01-13 13:37:00')); +SELECT 'dt != const dt64' FROM dt64test WHERE dt_column != toDateTime64('2020-01-13 13:37:00', 3); +SELECT 'dt != dt64' FROM dt64test WHERE dt_column != materialize(toDateTime64('2020-01-13 13:37:00', 3)); From c314d5393eb6be7da5c8cd739a167080f484b223 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 13 May 2021 20:44:55 +0300 Subject: [PATCH 035/202] Update CHANGELOG.md Co-authored-by: tavplubix --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a2c7cb66b8..b049227a6c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,6 @@ #### Backward Incompatible Change * Change comparison of integers and floating point numbers when integer is not exactly representable in the floating point data type. In new version comparison will return false as the rounding error will occur. Example: `9223372036854775808.0 != 9223372036854775808`, because the number `9223372036854775808` is not representable as floating point number exactly (and `9223372036854775808.0` is rounded to `9223372036854776000.0`). But in previous version the comparison will return as the numbers are equal, because if the floating point number `9223372036854776000.0` get converted back to UInt64, it will yield `9223372036854775808`. For the reference, the Python programming language also treats these numbers as equal. But this behaviour was dependend on CPU model (different results on AMD64 and AArch64 for some out-of-range numbers), so we make the comparison more precise. It will treat int and float numbers equal only if int is represented in floating point type exactly. [#22595](https://github.com/ClickHouse/ClickHouse/pull/22595) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)). #### New Feature From b4eb181378d082a80cc1abf5d39ab2afd61b0448 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 13 May 2021 20:51:16 +0300 Subject: [PATCH 036/202] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b049227a6c8..d3a8c13f5e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ * Allow to move more conditions to `PREWHERE` as it was before version 21.1. Insufficient number of moved condtions could lead to worse performance. [#23397](https://github.com/ClickHouse/ClickHouse/pull/23397) ([Anton Popov](https://github.com/CurtizJ)). * Remove support for `argMin` and `argMax` for single `Tuple` argument. The code was not memory-safe. The feature was added by mistake and it is confusing for people. These functions can be reintroduced under different names later. This fixes [#22384](https://github.com/ClickHouse/ClickHouse/issues/22384) and reverts [#17359](https://github.com/ClickHouse/ClickHouse/issues/17359). [#23393](https://github.com/ClickHouse/ClickHouse/pull/23393) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed incompatible constant expression generation during partition pruning based on virtual columns. This fixes https://github.com/ClickHouse/ClickHouse/pull/21401#discussion_r611888913. [#23366](https://github.com/ClickHouse/ClickHouse/pull/23366) ([Amos Bird](https://github.com/amosbird)). -* Fixed a bug in dict join with join_algorithm = 'auto'. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)). +* Fixed a crash when setting join_algorithm is set to 'auto' and Join is performed with a Dictionary. Close [#23002](https://github.com/ClickHouse/ClickHouse/issues/23002). [#23312](https://github.com/ClickHouse/ClickHouse/pull/23312) ([Vladimir](https://github.com/vdimir)). * Don't relax NOT conditions during partition pruning. This fixes [#23305](https://github.com/ClickHouse/ClickHouse/issues/23305) and [#21539](https://github.com/ClickHouse/ClickHouse/issues/21539). [#23310](https://github.com/ClickHouse/ClickHouse/pull/23310) ([Amos Bird](https://github.com/amosbird)). * Fix possible crash in case if `unknown packet` was received form remote query (with `async_socket_for_remote` enabled). Maybe fixes [#21167](https://github.com/ClickHouse/ClickHouse/issues/21167). [#23309](https://github.com/ClickHouse/ClickHouse/pull/23309) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)). From 2dd4aa7ea28cbf209ff925d29291b81cc5b842ca Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 13 May 2021 20:54:22 +0300 Subject: [PATCH 037/202] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3a8c13f5e4..e5e6b692bb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,7 +115,7 @@ * Avoid UB in *Log engines for rwlock unlock due to unlock from another thread. [#22583](https://github.com/ClickHouse/ClickHouse/pull/22583) ([Azat Khuzhin](https://github.com/azat)). * Fixed UB by unlocking the rwlock of the TinyLog from the same thread. [#22560](https://github.com/ClickHouse/ClickHouse/pull/22560) ([Azat Khuzhin](https://github.com/azat)). * Fixed deserialization of empty string without newline at end of TSV format. This closes [#20244](https://github.com/ClickHouse/ClickHouse/issues/20244). Possible workaround without version update: set `input_format_null_as_default` to zero. It was zero in old versions. [#22527](https://github.com/ClickHouse/ClickHouse/pull/22527) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Fix bug in partial merge join with `LowCardinality`. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)). +* Fixed wrong cast of a column of `LowCardinality` type in MergeJoin algorithm. Close [#22386](https://github.com/ClickHouse/ClickHouse/issues/22386), close [#22388](https://github.com/ClickHouse/ClickHouse/issues/22388). [#22510](https://github.com/ClickHouse/ClickHouse/pull/22510) ([Vladimir](https://github.com/vdimir)). * Fixed `ClickHouseDictionarySource` configuration loop. Closes [#14314](https://github.com/ClickHouse/ClickHouse/issues/14314). [#22479](https://github.com/ClickHouse/ClickHouse/pull/22479) ([Maksim Kita](https://github.com/kitaisreal)). * Fixed a race condition in HedgedConnections which leads to crash. This fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)). * Buffer overflow (on read) was possible in `tokenbf_v1` full text index. The excessive bytes are not used but the read operation may lead to crash in rare cases. This closes [#19233](https://github.com/ClickHouse/ClickHouse/issues/19233). [#22421](https://github.com/ClickHouse/ClickHouse/pull/22421) ([alexey-milovidov](https://github.com/alexey-milovidov)). From 678705b00403afb402b0a5ab840d97d610b391e8 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:49:03 +0300 Subject: [PATCH 038/202] Update docs/ru/sql-reference/functions/ip-address-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/ip-address-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index 10ded819fef..b02d45d7667 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -399,7 +399,7 @@ SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0 Проверяет, попадает ли IP адрес в интервал, заданный в нотации [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). -**Syntax** +**Синтаксис** ``` sql isIPAddressInRange(address, prefix) From cd10e84f5cd7b557c8b46f38d4c8a25508396d50 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:49:30 +0300 Subject: [PATCH 039/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index d86de71ec72..1e5788c70a4 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -369,7 +369,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut; ## reinterpretAsUUID {#reinterpretasuuid} -Функция принимает строку из 16 байт и интерпретирует ее байты в порядок байт от старшего к младшему. Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количеством нулевых байтов с конца. Если строка длиннее, чем 16 байтов, то лишние байты с конца игнорируются. +Функция принимает строку из 16 байт и интерпретирует ее байты в порядок от старшего к младшему. Если строка имеет недостаточную длину, то функция работает так, как будто строка дополнена необходимым количеством нулевых байтов с конца. Если строка длиннее, чем 16 байтов, то лишние байты с конца игнорируются. **Синтаксис** @@ -1336,4 +1336,3 @@ FROM numbers(3); │ 2,"good" │ └───────────────────────────────────────────┘ ``` - From 113063209327ebb9126a63c884bbde89afde70b4 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:49:42 +0300 Subject: [PATCH 040/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 1e5788c70a4..cf7eb760378 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -467,7 +467,7 @@ SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint, Поддерживается также синтаксис `CAST(x AS t)`. !!! note "Примечание" - Обратите внимание, что если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255. + Если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255. **Синтаксис** From 79f8265897658f435552684d090638068e5436ca Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:50:38 +0300 Subject: [PATCH 041/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index cf7eb760378..d75fb03d45a 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1081,7 +1081,7 @@ parseDateTime64BestEffort(time_string [, precision [, time_zone]]) **Аргументы** - `time_string` — строка, содержащая дату или дату со временем, которые нужно преобразовать. [String](../../sql-reference/data-types/string.md). -- `precision` — `3` для миллисекунд, `6` для микросекунд. По умолчанию `3`. Необязательный. [UInt8](../../sql-reference/data-types/int-uint.md). +- `precision` — требуемая точность: `3` — для миллисекунд, `6` — для микросекунд. По умолчанию — `3`. Необязательный. [UInt8](../../sql-reference/data-types/int-uint.md). - `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). Разбирает значение `time_string` в зависимости от часового пояса. Необязательный. [String](../../sql-reference/data-types/string.md). **Возвращаемое значение** From f2307b23e3a88d1ca2d70da43665af3309583096 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:50:56 +0300 Subject: [PATCH 042/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index d75fb03d45a..0c55b183f69 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1169,7 +1169,7 @@ SELECT toLowCardinality('1'); Преобразует значение `DateTime64` в значение `Int64` с фиксированной точностью менее одной секунды. Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. -!!! note "Примечание" +!!! info "Примечание" Обратите внимание, что возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`. **Синтаксис** From 8bdd08b63d0270458fbb680b39dddf6c45b809bf Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:51:01 +0300 Subject: [PATCH 043/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 0c55b183f69..9a25facb9bb 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1170,7 +1170,7 @@ SELECT toLowCardinality('1'); Входное значение округляется соответствующим образом вверх или вниз в зависимости от его точности. !!! info "Примечание" - Обратите внимание, что возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`. + Возвращаемое значение — это временная метка в UTC, а не в часовом поясе `DateTime64`. **Синтаксис** From 4c7c2ac8637a1894a5a4896318e6e91a34b8eaf1 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Thu, 13 May 2021 21:51:09 +0300 Subject: [PATCH 044/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 9a25facb9bb..c318dded88c 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -466,7 +466,7 @@ SELECT reinterpret(toInt8(-1), 'UInt8') as int_to_uint, Поддерживается также синтаксис `CAST(x AS t)`. -!!! note "Примечание" +!!! warning "Предупреждение" Если значение `x` не может быть преобразовано к типу `T`, возникает переполнение. Например, `CAST(-1, 'UInt8')` возвращает 255. **Синтаксис** From 53b9ead1a9f224409576c51cd30153fe9a467f01 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 14 May 2021 00:12:36 +0300 Subject: [PATCH 045/202] Update CHANGELOG.md Co-authored-by: Azat Khuzhin --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5e6b692bb0..f3dd92f4d32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ * Improved performance of `intDiv` by dynamic dispatch for AVX2. This closes [#22314](https://github.com/ClickHouse/ClickHouse/issues/22314). [#23000](https://github.com/ClickHouse/ClickHouse/pull/23000) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Improved performance of reading from `ArrowStream` input format for sources other then local file (e.g. URL). [#22673](https://github.com/ClickHouse/ClickHouse/pull/22673) ([nvartolomei](https://github.com/nvartolomei)). * Disabled compression by default when interacting with localhost (with clickhouse-client or server to server with distributed queries) via native protocol. It may improve performance of some import/export operations. This closes [#22234](https://github.com/ClickHouse/ClickHouse/issues/22234). [#22237](https://github.com/ClickHouse/ClickHouse/pull/22237) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Exclude values that does not belong to the shard from right part of IN section for distributed queries. [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)). +* Exclude values that does not belong to the shard from right part of IN section for distributed queries (under `optimize_skip_unused_shards_rewrite_in`, enabled by default, since it still requires `optimize_skip_unused_shards`). [#21511](https://github.com/ClickHouse/ClickHouse/pull/21511) ([Azat Khuzhin](https://github.com/azat)). * Improved performance of reading a subset of columns with File-like table engine and column-oriented format like Parquet, Arrow or ORC. This closes [#issue:20129](https://github.com/ClickHouse/ClickHouse/issues/20129). [#21302](https://github.com/ClickHouse/ClickHouse/pull/21302) ([keenwolf](https://github.com/keen-wolf)). #### Improvement From ba754b51f95992317dbb93f2ac02061af96840bd Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 14 May 2021 00:54:43 +0300 Subject: [PATCH 046/202] Update CHANGELOG.md Co-authored-by: Azat Khuzhin --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3dd92f4d32..c9d5511f383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,7 +86,6 @@ * Fixed very rare race condition on background cleanup of old blocks. It might cause a block not to be deduplicated if it's too close to the end of deduplication window. [#23301](https://github.com/ClickHouse/ClickHouse/pull/23301) ([tavplubix](https://github.com/tavplubix)). * Fixed very rare (distributed) race condition between creation and removal of ReplicatedMergeTree tables. It might cause exceptions like `node doesn't exist` on attempt to create replicated table. Fixes [#21419](https://github.com/ClickHouse/ClickHouse/issues/21419). [#23294](https://github.com/ClickHouse/ClickHouse/pull/23294) ([tavplubix](https://github.com/tavplubix)). * Fixed simple key dictionary from DDL creation if primary key is not first attribute. Fixes [#23236](https://github.com/ClickHouse/ClickHouse/issues/23236). [#23262](https://github.com/ClickHouse/ClickHouse/pull/23262) ([Maksim Kita](https://github.com/kitaisreal)). -* Add type conversion for `optimize_skip_unused_shards_rewrite_in` (fixes `use-of-uninitialized-value` with `optimize_skip_unused_shards_rewrite_in`). [#23219](https://github.com/ClickHouse/ClickHouse/pull/23219) ([Azat Khuzhin](https://github.com/azat)). * Fixed reading from ODBC when there are many long column names in a table. Closes [#8853](https://github.com/ClickHouse/ClickHouse/issues/8853). [#23215](https://github.com/ClickHouse/ClickHouse/pull/23215) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fixed missing semicolon in exception message. The user may find this exception message unpleasant to read. [#23208](https://github.com/ClickHouse/ClickHouse/pull/23208) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fixed missing whitespace in some exception messages about `LowCardinality` type. [#23207](https://github.com/ClickHouse/ClickHouse/pull/23207) ([alexey-milovidov](https://github.com/alexey-milovidov)). From 1a049c396dff10c8fd77c522949885974e251657 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Fri, 14 May 2021 09:30:51 +0300 Subject: [PATCH 047/202] Update docs/en/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/functions/type-conversion-functions.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 8a477b79836..2b3841556d8 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1101,7 +1101,7 @@ UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t -FORMAT PrettyCompactMonoBlcok; +FORMAT PrettyCompactMonoBlock; ``` Result: @@ -1337,4 +1337,3 @@ Result: │ 2,"good" │ └───────────────────────────────────────────┘ ``` - From 1b7d2db4c653163556fd31682f0daa4ca1a01612 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Fri, 14 May 2021 09:30:59 +0300 Subject: [PATCH 048/202] Update docs/en/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 2b3841556d8..fea415cf1ac 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1170,7 +1170,7 @@ Result: Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision. -!!! note "Note" +!!! info "Note" Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`. **Syntax** From e3b9fde94d662b237a9a65ccdf79b5700dbad90a Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Fri, 14 May 2021 09:31:06 +0300 Subject: [PATCH 049/202] Update docs/en/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index fea415cf1ac..65dd9c90ec4 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1171,7 +1171,7 @@ Result: Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision. !!! info "Note" - Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`. + The output value is a timestamp in UTC, not in the timezone of `DateTime64`. **Syntax** From be15c96de6b72516eb52b09f0e73ffb7bdf64799 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Fri, 14 May 2021 09:31:14 +0300 Subject: [PATCH 050/202] Update docs/ru/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index c318dded88c..c55d0b7013f 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1100,7 +1100,7 @@ UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',6) AS a, toTypeName(a) AS t UNION ALL SELECT parseDateTime64BestEffort('2021-01-01 01:01:00.12346',3,'Europe/Moscow') AS a, toTypeName(a) AS t -FORMAT PrettyCompactMonoBlcok; +FORMAT PrettyCompactMonoBlock; ``` Результат: From 765bc28f61333664fdb3204eb5caebbde8fd5272 Mon Sep 17 00:00:00 2001 From: Evgenia Sudarikova <56156889+otrazhenia@users.noreply.github.com> Date: Fri, 14 May 2021 09:31:21 +0300 Subject: [PATCH 051/202] Update docs/en/sql-reference/functions/type-conversion-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/functions/type-conversion-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 65dd9c90ec4..8cd858c2a10 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1082,7 +1082,7 @@ parseDateTime64BestEffort(time_string [, precision [, time_zone]]) **Parameters** - `time_string` — String containing a date or date with time to convert. [String](../../sql-reference/data-types/string.md). -- `precision` — `3` for milliseconds, `6` for microseconds. Default `3`. Optional. [UInt8](../../sql-reference/data-types/int-uint.md). +- `precision` — Required precision. `3` — for milliseconds, `6` — for microseconds. Default — `3`. Optional. [UInt8](../../sql-reference/data-types/int-uint.md). - `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). **Returned value** From 55c573f9205220fcd379b82a7508e4ce0e18f8a6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 14 May 2021 07:11:32 +0000 Subject: [PATCH 052/202] Fix --- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 8ec55c1b1c1..ccb24d5db90 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -458,6 +458,9 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, AMQP::TcpChannel & rabbit_chann AMQP::Table queue_settings; + std::unordered_set integer_settings = {"x-max-length", "x-max-length-bytes", "x-message-ttl", "x-expires", "x-priority", "x-max-priority"}; + std::unordered_set string_settings = {"x-overflow", "x-dead-letter-exchange", "x-queue-type"}; + /// Check user-defined settings. if (!queue_settings_list.empty()) { @@ -465,13 +468,12 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, AMQP::TcpChannel & rabbit_chann { Strings setting_values; splitInto<'='>(setting_values, setting); - assert(setting_values.size() == 2); + if (setting_values.size() != 2) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid settings string: {}", setting); + String key = setting_values[0], value = setting_values[1]; - std::unordered_set integer_settings = {"x-max-length", "x-max-length-bytes", "x-message-ttl", "x-expires", "x-priority", "x-max-priority"}; - std::unordered_set string_settings = {"x-overflow", "x-dead-letter-exchange", "x-queue-type"}; - - if (integer_settings.find(key) != integer_settings.end()) + if (integer_settings.contains(key)) queue_settings[key] = parse(value); else if (string_settings.find(key) != string_settings.end()) queue_settings[key] = value; From 81288e4b5071e153ad318987b60b9a1d4c7d8766 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Fri, 14 May 2021 14:07:11 +0300 Subject: [PATCH 053/202] Apply suggestions from code review --- .../aggregate-functions/parametric-functions.md | 2 +- .../functions/type-conversion-functions.md | 2 +- .../functions/type-conversion-functions.md | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 2a221bbb6eb..0edb1601023 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -253,7 +253,7 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN) **Parameters** -- `window` — Length of the sliding window, it is the time interval between the first condition and last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`. +- `window` — Length of the sliding window, it is the time interval between the first and the last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`. - `mode` — It is an optional argument. One or more modes can be set. - `'strict'` — If same condition holds for sequence of events then such non-unique events would be skipped. - `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2. diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 8cd858c2a10..661469e6901 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -373,7 +373,7 @@ This function accepts a number or date or date with time, and returns a FixedStr ## reinterpretAsUUID {#reinterpretasuuid} -Accepts 16 bytes string and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the functions works as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored. +Accepts 16 bytes string and returns UUID containing bytes representing the corresponding value in network byte order (big-endian). If the string isn't long enough, the function works as if the string is padded with the necessary number of null bytes to the end. If the string longer than 16 bytes, the extra bytes at the end are ignored. **Syntax** diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index c55d0b7013f..2226c90525d 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -440,7 +440,7 @@ reinterpret(x, type) **Возвращаемое значение** -- Конечный тип данных. +- Значение конечного типа данных. **Примеры** @@ -482,7 +482,7 @@ CAST(x, T) **Возвращаемое значение** -- Конечный тип данных. +- Значение конечного типа данных. **Примеры** @@ -894,7 +894,7 @@ AS parseDateTimeBestEffortUS; ## parseDateTimeBestEffortOrZero {#parsedatetimebesteffortorzero} ## parseDateTime32BestEffortOrZero {#parsedatetime32besteffortorzero} -Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевую дату или нулевую дату и время в случае, если получает формат даты, который не может быть обработан. +Работает аналогично функции [parseDateTimeBestEffort](#parsedatetimebesteffort), но возвращает нулевое значение, если формат даты не может быть обработан. ## parseDateTimeBestEffortUSOrNull {#parsedatetimebesteffortusornull} @@ -1116,11 +1116,11 @@ FORMAT PrettyCompactMonoBlock; ## parseDateTime64BestEffortOrNull {#parsedatetime32besteffortornull} -Работает аналогично функции [parseDateTime64BestEffort](#parsedatetime64besteffort), но возвращает `NULL` в случае, если встречает формат даты, который не может обработать. +Работает аналогично функции [parseDateTime64BestEffort](#parsedatetime64besteffort), но возвращает `NULL`, если формат даты не может быть обработан. ## parseDateTime64BestEffortOrZero {#parsedatetime64besteffortorzero} -Работает аналогично функции [parseDateTime64BestEffort](#parsedatetimebesteffort), но возвращает нулевую дату и время в случае, если встречает формат даты, который не может обработать. +Работает аналогично функции [parseDateTime64BestEffort](#parsedatetimebesteffort), но возвращает нулевую дату и время, если формат даты не может быть обработан. ## toLowCardinality {#tolowcardinality} From 123715300aef2798195b109b560d61791a8bb5af Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 14 May 2021 15:30:49 +0300 Subject: [PATCH 054/202] Use keeper in integration tests --- .../runner/compose/docker_compose_keeper.yml | 92 ++++++++++++++++++ tests/integration/helpers/cluster.py | 96 ++++++++++++------- tests/integration/helpers/keeper_config1.xml | 38 ++++++++ tests/integration/helpers/keeper_config2.xml | 38 ++++++++ tests/integration/helpers/keeper_config3.xml | 38 ++++++++ 5 files changed, 270 insertions(+), 32 deletions(-) create mode 100644 docker/test/integration/runner/compose/docker_compose_keeper.yml create mode 100644 tests/integration/helpers/keeper_config1.xml create mode 100644 tests/integration/helpers/keeper_config2.xml create mode 100644 tests/integration/helpers/keeper_config3.xml diff --git a/docker/test/integration/runner/compose/docker_compose_keeper.yml b/docker/test/integration/runner/compose/docker_compose_keeper.yml new file mode 100644 index 00000000000..14d878a8271 --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_keeper.yml @@ -0,0 +1,92 @@ +version: '2.3' +services: + zoo1: + image: ${image:-} + restart: always + user: ${user:-} + volumes: + - type: bind + source: ${keeper_binary:-} + target: /usr/bin/clickhouse + - type: bind + source: ${keeper_config_dir1:-} + target: /etc/clickhouse-keeper + - type: bind + source: ${keeper_logs_dir1:-} + target: /var/log/clickhouse-keeper + - type: ${keeper_fs:-tmpfs} + source: ${keeper_db_dir1:-} + target: /var/lib/clickhouse/coordination + entrypoint: "clickhouse keeper --config=/etc/clickhouse-keeper/keeper_config1.xml --log-file=/var/log/clickhouse-keeper/clickhouse-keeper.log --errorlog-file=/var/log/clickhouse-keeper/clickhouse-keeper.err.log" + cap_add: + - SYS_PTRACE + - NET_ADMIN + - IPC_LOCK + - SYS_NICE + security_opt: + - label:disable + dns_opt: + - attempts:2 + - timeout:1 + - inet6 + - rotate + zoo2: + image: ${image:-} + restart: always + user: ${user:-} + volumes: + - type: bind + source: ${keeper_binary:-} + target: /usr/bin/clickhouse + - type: bind + source: ${keeper_config_dir2:-} + target: /etc/clickhouse-keeper + - type: bind + source: ${keeper_logs_dir2:-} + target: /var/log/clickhouse-keeper + - type: ${keeper_fs:-tmpfs} + source: ${keeper_db_dir2:-} + target: /var/lib/clickhouse/coordination + entrypoint: "clickhouse keeper --config=/etc/clickhouse-keeper/keeper_config2.xml --log-file=/var/log/clickhouse-keeper/clickhouse-keeper.log --errorlog-file=/var/log/clickhouse-keeper/clickhouse-keeper.err.log" + cap_add: + - SYS_PTRACE + - NET_ADMIN + - IPC_LOCK + - SYS_NICE + security_opt: + - label:disable + dns_opt: + - attempts:2 + - timeout:1 + - inet6 + - rotate + zoo3: + image: ${image:-} + restart: always + user: ${user:-} + volumes: + - type: bind + source: ${keeper_binary:-} + target: /usr/bin/clickhouse + - type: bind + source: ${keeper_config_dir3:-} + target: /etc/clickhouse-keeper + - type: bind + source: ${keeper_logs_dir3:-} + target: /var/log/clickhouse-keeper + - type: ${keeper_fs:-tmpfs} + source: ${keeper_db_dir3:-} + target: /var/lib/clickhouse/coordination + entrypoint: "clickhouse keeper --config=/etc/clickhouse-keeper/keeper_config3.xml --log-file=/var/log/clickhouse-keeper/clickhouse-keeper.log --errorlog-file=/var/log/clickhouse-keeper/clickhouse-keeper.err.log" + cap_add: + - SYS_PTRACE + - NET_ADMIN + - IPC_LOCK + - SYS_NICE + security_opt: + - label:disable + dns_opt: + - attempts:2 + - timeout:1 + - inet6 + - rotate diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index ed28c3a7fc4..da0898b7c23 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -37,6 +37,7 @@ DEFAULT_ENV_NAME = 'env_file' SANITIZER_SIGN = "==================" +USE_KEEPER = True def _create_env_file(path, variables, fname=DEFAULT_ENV_NAME): full_path = os.path.join(path, fname) @@ -202,6 +203,7 @@ class ClickHouseCluster: self.docker_client = None self.is_up = False + self.env = os.environ.copy() print("CLUSTER INIT base_config_dir:{}".format(self.base_config_dir)) def get_client_cmd(self): @@ -291,7 +293,10 @@ class ClickHouseCluster: cmds = [] if with_zookeeper and not self.with_zookeeper: if not zookeeper_docker_compose_path: - zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_zookeeper.yml') + if USE_KEEPER: + zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_keeper.yml') + else: + zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_zookeeper.yml') self.with_zookeeper = True self.zookeeper_use_tmpfs = zookeeper_use_tmpfs @@ -672,29 +677,59 @@ class ClickHouseCluster: except: pass - clickhouse_pull_cmd = self.base_cmd + ['pull'] - print(f"Pulling images for {self.base_cmd}") - retry_exception(10, 5, subprocess_check_call, Exception, clickhouse_pull_cmd) + #clickhouse_pull_cmd = self.base_cmd + ['pull'] + #print(f"Pulling images for {self.base_cmd}") + #retry_exception(10, 5, subprocess_check_call, Exception, clickhouse_pull_cmd) self.docker_client = docker.from_env(version=self.docker_api_version) common_opts = ['up', '-d'] if self.with_zookeeper and self.base_zookeeper_cmd: - print('Setup ZooKeeper') - env = os.environ.copy() - if not self.zookeeper_use_tmpfs: - env['ZK_FS'] = 'bind' - for i in range(1, 4): - zk_data_path = self.instances_dir + '/zkdata' + str(i) - zk_log_data_path = self.instances_dir + '/zklog' + str(i) - if not os.path.exists(zk_data_path): - os.mkdir(zk_data_path) - if not os.path.exists(zk_log_data_path): - os.mkdir(zk_log_data_path) - env['ZK_DATA' + str(i)] = zk_data_path - env['ZK_DATA_LOG' + str(i)] = zk_log_data_path - run_and_check(self.base_zookeeper_cmd + common_opts, env=env) + if USE_KEEPER: + print('Setup Keeper') + binary_path = self.server_bin_path + if binary_path.endswith('-server'): + binary_path = binary_path[:-len('-server')] + + self.env['keeper_binary'] = binary_path + self.env['image'] = "yandex/clickhouse-integration-test:" + self.docker_base_tag + self.env['user'] = str(os.getuid()) + if not self.zookeeper_use_tmpfs: + self.env['keeper_fs'] = 'bind' + + for i in range (1, 4): + instance_dir = p.join(self.instances_dir, f"keeper{i}") + logs_dir = p.join(instance_dir, "logs") + configs_dir = p.join(instance_dir, "configs") + coordination_dir = p.join(instance_dir, "coordination") + if not os.path.exists(instance_dir): + os.mkdir(instance_dir) + os.mkdir(configs_dir) + os.mkdir(logs_dir) + if not self.zookeeper_use_tmpfs: + os.mkdir(coordination_dir) + shutil.copy(os.path.join(HELPERS_DIR, f'keeper_config{i}.xml'), configs_dir) + + self.env[f'keeper_logs_dir{i}'] = p.abspath(logs_dir) + self.env[f'keeper_config_dir{i}'] = p.abspath(configs_dir) + if not self.zookeeper_use_tmpfs: + self.env[f'keeper_db_dir{i}'] = p.abspath(coordination_dir) + else: + print('Setup ZooKeeper') + if not self.zookeeper_use_tmpfs: + self.env['ZK_FS'] = 'bind' + for i in range(1, 4): + zk_data_path = self.instances_dir + '/zkdata' + str(i) + zk_log_data_path = self.instances_dir + '/zklog' + str(i) + if not os.path.exists(zk_data_path): + os.mkdir(zk_data_path) + if not os.path.exists(zk_log_data_path): + os.mkdir(zk_log_data_path) + self.env['ZK_DATA' + str(i)] = zk_data_path + self.env['ZK_DATA_LOG' + str(i)] = zk_log_data_path + + run_and_check(self.base_zookeeper_cmd + common_opts, env=self.env) for command in self.pre_zookeeper_commands: self.run_kazoo_commands_with_retries(command, repeats=5) self.wait_zookeeper_to_start(120) @@ -731,9 +766,8 @@ class ClickHouseCluster: if self.with_kerberized_kafka and self.base_kerberized_kafka_cmd: print('Setup kerberized kafka') - env = os.environ.copy() - env['KERBERIZED_KAFKA_DIR'] = instance.path + '/' - run_and_check(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes'], env=env) + self.env['KERBERIZED_KAFKA_DIR'] = instance.path + '/' + run_and_check(self.base_kerberized_kafka_cmd + common_opts + ['--renew-anon-volumes'], env=self.env) self.kerberized_kafka_docker_id = self.get_instance_docker_id('kerberized_kafka1') if self.with_rabbitmq and self.base_rabbitmq_cmd: subprocess_check_call(self.base_rabbitmq_cmd + common_opts + ['--renew-anon-volumes']) @@ -747,9 +781,8 @@ class ClickHouseCluster: if self.with_kerberized_hdfs and self.base_kerberized_hdfs_cmd: print('Setup kerberized HDFS') - env = os.environ.copy() - env['KERBERIZED_HDFS_DIR'] = instance.path + '/' - run_and_check(self.base_kerberized_hdfs_cmd + common_opts, env=env) + self.env['KERBERIZED_HDFS_DIR'] = instance.path + '/' + run_and_check(self.base_kerberized_hdfs_cmd + common_opts, env=self.env) self.make_hdfs_api(kerberized=True) self.wait_hdfs_to_start(timeout=300) @@ -764,23 +797,22 @@ class ClickHouseCluster: time.sleep(10) if self.with_minio and self.base_minio_cmd: - env = os.environ.copy() prev_ca_certs = os.environ.get('SSL_CERT_FILE') if self.minio_certs_dir: minio_certs_dir = p.join(self.base_dir, self.minio_certs_dir) - env['MINIO_CERTS_DIR'] = minio_certs_dir + self.env['MINIO_CERTS_DIR'] = minio_certs_dir # Minio client (urllib3) uses SSL_CERT_FILE for certificate validation. os.environ['SSL_CERT_FILE'] = p.join(minio_certs_dir, 'public.crt') else: # Attach empty certificates directory to ensure non-secure mode. minio_certs_dir = p.join(self.instances_dir, 'empty_minio_certs_dir') os.mkdir(minio_certs_dir) - env['MINIO_CERTS_DIR'] = minio_certs_dir + self.env['MINIO_CERTS_DIR'] = minio_certs_dir minio_start_cmd = self.base_minio_cmd + common_opts logging.info("Trying to create Minio instance by command %s", ' '.join(map(str, minio_start_cmd))) - run_and_check(minio_start_cmd, env=env) + run_and_check(minio_start_cmd, env=self.env) try: logging.info("Trying to connect to Minio...") @@ -799,7 +831,7 @@ class ClickHouseCluster: clickhouse_start_cmd = self.base_cmd + ['up', '-d', '--no-recreate'] print(("Trying to create ClickHouse instance by command %s", ' '.join(map(str, clickhouse_start_cmd)))) - subprocess_check_call(clickhouse_start_cmd) + run_and_check(clickhouse_start_cmd, env=self.env) print("ClickHouse instance created") start_deadline = time.time() + 20.0 # seconds @@ -825,7 +857,7 @@ class ClickHouseCluster: sanitizer_assert_instance = None with open(self.docker_logs_path, "w+") as f: try: - subprocess.check_call(self.base_cmd + ['logs'], stdout=f) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call(self.base_cmd + ['logs'], env=self.env, stdout=f) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL except Exception as e: print("Unable to get logs from docker.") f.seek(0) @@ -836,14 +868,14 @@ class ClickHouseCluster: if kill: try: - subprocess_check_call(self.base_cmd + ['stop', '--timeout', '20']) + run_and_check(self.base_cmd + ['stop', '--timeout', '20'], env=self.env) except Exception as e: print("Kill command failed during shutdown. {}".format(repr(e))) print("Trying to kill forcefully") subprocess_check_call(self.base_cmd + ['kill']) try: - subprocess_check_call(self.base_cmd + ['down', '--volumes', '--remove-orphans']) + run_and_check(self.base_cmd + ['down', '--volumes', '--remove-orphans'], env=self.env) except Exception as e: print("Down + remove orphans failed durung shutdown. {}".format(repr(e))) diff --git a/tests/integration/helpers/keeper_config1.xml b/tests/integration/helpers/keeper_config1.xml new file mode 100644 index 00000000000..2d2f66cbc39 --- /dev/null +++ b/tests/integration/helpers/keeper_config1.xml @@ -0,0 +1,38 @@ + + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 1 + + + 10000 + 30000 + false + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + diff --git a/tests/integration/helpers/keeper_config2.xml b/tests/integration/helpers/keeper_config2.xml new file mode 100644 index 00000000000..81976015df1 --- /dev/null +++ b/tests/integration/helpers/keeper_config2.xml @@ -0,0 +1,38 @@ + + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 2 + + + 10000 + 30000 + false + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + diff --git a/tests/integration/helpers/keeper_config3.xml b/tests/integration/helpers/keeper_config3.xml new file mode 100644 index 00000000000..07a9e5fc6b7 --- /dev/null +++ b/tests/integration/helpers/keeper_config3.xml @@ -0,0 +1,38 @@ + + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 3 + + + 10000 + 30000 + false + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + From de75d9c887c7e4775a595aca3eb4a98f775ccd09 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 14 May 2021 16:44:32 +0300 Subject: [PATCH 055/202] Revert accident changes --- tests/integration/helpers/cluster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index da0898b7c23..1edee6216a6 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -677,9 +677,9 @@ class ClickHouseCluster: except: pass - #clickhouse_pull_cmd = self.base_cmd + ['pull'] - #print(f"Pulling images for {self.base_cmd}") - #retry_exception(10, 5, subprocess_check_call, Exception, clickhouse_pull_cmd) + clickhouse_pull_cmd = self.base_cmd + ['pull'] + print(f"Pulling images for {self.base_cmd}") + retry_exception(10, 5, subprocess_check_call, Exception, clickhouse_pull_cmd) self.docker_client = docker.from_env(version=self.docker_api_version) From 18bf681a83066f8f75537326730bddb12c67ffa4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 14 May 2021 18:35:30 +0300 Subject: [PATCH 056/202] Fix pull --- .../integration/runner/compose/docker_compose_keeper.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/integration/runner/compose/docker_compose_keeper.yml b/docker/test/integration/runner/compose/docker_compose_keeper.yml index 14d878a8271..39d4a9c53d9 100644 --- a/docker/test/integration/runner/compose/docker_compose_keeper.yml +++ b/docker/test/integration/runner/compose/docker_compose_keeper.yml @@ -1,7 +1,7 @@ version: '2.3' services: zoo1: - image: ${image:-} + image: ${image:-yandex/clickhouse-integration-test} restart: always user: ${user:-} volumes: @@ -31,7 +31,7 @@ services: - inet6 - rotate zoo2: - image: ${image:-} + image: ${image:-yandex/clickhouse-integration-test} restart: always user: ${user:-} volumes: @@ -61,7 +61,7 @@ services: - inet6 - rotate zoo3: - image: ${image:-} + image: ${image:-yandex/clickhouse-integration-test} restart: always user: ${user:-} volumes: From 26be39f4197e8282c25c73fef5970be92a84a7fe Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 11 May 2021 21:59:40 +0300 Subject: [PATCH 057/202] CompileExpressions comparison function constant case fix --- src/Interpreters/ExpressionJIT.cpp | 13 ++++++++++++ ...5_jit_comparison_constant_result.reference | 8 +++++++ .../01855_jit_comparison_constant_result.sql | 21 +++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 4169d95bfa4..3d565dd384b 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -328,6 +328,13 @@ static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Node & node, c NameGreaterOrEquals::name }; + /** Comparision operator is special case for ActionDAG compilation + * Its result can be constant and we can understand that only during Function execute call. + * It can be a problem if two DAGs with compare function are analyzed, but in first DAG comparison + * function is compiled, in second DAG it is not compiled. + * There will be error because of block headers mismatch. + */ + auto it = comparison_functions.find(impl.getName()); if (it == comparison_functions.end()) return false; @@ -335,6 +342,12 @@ static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Node & node, c const auto * lhs_node = node.children[0]; const auto * rhs_node = node.children[1]; + while (lhs_node->type == ActionsDAG::ActionType::ALIAS) + lhs_node = lhs_node->children[0]; + + while (rhs_node->type == ActionsDAG::ActionType::ALIAS) + rhs_node = rhs_node->children[0]; + return lhs_node == rhs_node && !isTuple(lhs_node->result_type); } diff --git a/tests/queries/0_stateless/01855_jit_comparison_constant_result.reference b/tests/queries/0_stateless/01855_jit_comparison_constant_result.reference index a9e2f17562a..e97edac16d6 100644 --- a/tests/queries/0_stateless/01855_jit_comparison_constant_result.reference +++ b/tests/queries/0_stateless/01855_jit_comparison_constant_result.reference @@ -1,3 +1,11 @@ +ComparisionOperator column with same column +1 +1 +1 +1 +1 +1 +ComparisionOperator column with alias on same column 1 1 1 diff --git a/tests/queries/0_stateless/01855_jit_comparison_constant_result.sql b/tests/queries/0_stateless/01855_jit_comparison_constant_result.sql index b8d06e218e0..51cf9aa1d17 100644 --- a/tests/queries/0_stateless/01855_jit_comparison_constant_result.sql +++ b/tests/queries/0_stateless/01855_jit_comparison_constant_result.sql @@ -1,6 +1,8 @@ SET compile_expressions = 1; SET min_count_to_compile_expression = 0; +SELECT 'ComparisionOperator column with same column'; + DROP TABLE IF EXISTS test_table; CREATE TABLE test_table (a UInt64) ENGINE = MergeTree() ORDER BY tuple(); INSERT INTO test_table VALUES (1); @@ -13,3 +15,22 @@ SELECT test_table.a FROM test_table ORDER BY (test_table.a <= test_table.a) + 1; SELECT test_table.a FROM test_table ORDER BY (test_table.a == test_table.a) + 1; SELECT test_table.a FROM test_table ORDER BY (test_table.a != test_table.a) + 1; + +DROP TABLE test_table; + +SELECT 'ComparisionOperator column with alias on same column'; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table (a UInt64, b ALIAS a, c ALIAS b) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO test_table VALUES (1); + +SELECT test_table.a FROM test_table ORDER BY (test_table.a > test_table.b) + 1 AND (test_table.a > test_table.c) + 1; +SELECT test_table.a FROM test_table ORDER BY (test_table.a >= test_table.b) + 1 AND (test_table.a >= test_table.c) + 1; + +SELECT test_table.a FROM test_table ORDER BY (test_table.a < test_table.b) + 1 AND (test_table.a < test_table.c) + 1; +SELECT test_table.a FROM test_table ORDER BY (test_table.a <= test_table.b) + 1 AND (test_table.a <= test_table.c) + 1; + +SELECT test_table.a FROM test_table ORDER BY (test_table.a == test_table.b) + 1 AND (test_table.a == test_table.c) + 1; +SELECT test_table.a FROM test_table ORDER BY (test_table.a != test_table.b) + 1 AND (test_table.a != test_table.c) + 1; + +DROP TABLE test_table; From f42b25e65c9d385d4a47b5711fb7678e8edbe6f3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 12 May 2021 09:52:40 +0300 Subject: [PATCH 058/202] Update ExpressionJIT.cpp --- src/Interpreters/ExpressionJIT.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 3d565dd384b..09b2130e4bb 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -316,7 +316,7 @@ static bool isCompilableConstant(const ActionsDAG::Node & node) return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type) && node.allow_constant_folding; } -static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Node & node, const IFunctionBase & impl) +static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Noomparision de & node, const IFunctionBase & impl) { static std::unordered_set comparison_functions { @@ -328,7 +328,7 @@ static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Node & node, c NameGreaterOrEquals::name }; - /** Comparision operator is special case for ActionDAG compilation + /** Comparison operator is special case for ActionDAG compilation * Its result can be constant and we can understand that only during Function execute call. * It can be a problem if two DAGs with compare function are analyzed, but in first DAG comparison * function is compiled, in second DAG it is not compiled. From 18894005ac80fb799e789c0f6f5d3765f2c25103 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 12 May 2021 10:36:09 +0300 Subject: [PATCH 059/202] Update ExpressionJIT.cpp --- src/Interpreters/ExpressionJIT.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 09b2130e4bb..b26092154a0 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -316,7 +316,7 @@ static bool isCompilableConstant(const ActionsDAG::Node & node) return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type) && node.allow_constant_folding; } -static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Noomparision de & node, const IFunctionBase & impl) +static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Node & node, const IFunctionBase & impl) { static std::unordered_set comparison_functions { From 0b4b5eb80883f7100979f03a49064195ccbb7745 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 12 May 2021 17:43:37 +0300 Subject: [PATCH 060/202] FunctionComparison remove constant result for non constant arguments optimization --- src/Functions/FunctionsComparison.h | 11 +++++++-- src/Interpreters/ExpressionJIT.cpp | 38 ----------------------------- 2 files changed, 9 insertions(+), 40 deletions(-) diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index f724915b3bc..f7c45558750 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -1148,17 +1148,24 @@ public: /// NOTE: We consider NaN comparison to be implementation specific (and in our implementation NaNs are sometimes equal sometimes not). if (left_type->equals(*right_type) && !left_type->isNullable() && !isTuple(left_type) && col_left_untyped == col_right_untyped) { + ColumnPtr result_column; + /// Always true: =, <=, >= if constexpr (IsOperation::equals || IsOperation::less_or_equals || IsOperation::greater_or_equals) { - return DataTypeUInt8().createColumnConst(input_rows_count, 1u); + result_column = DataTypeUInt8().createColumnConst(input_rows_count, 1u); } else { - return DataTypeUInt8().createColumnConst(input_rows_count, 0u); + result_column = DataTypeUInt8().createColumnConst(input_rows_count, 0u); } + + if (!isColumnConst(*col_left_untyped)) + result_column = result_column->convertToFullColumnIfConst(); + + return result_column; } WhichDataType which_left{left_type}; diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index b26092154a0..3b23e232184 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -316,41 +316,6 @@ static bool isCompilableConstant(const ActionsDAG::Node & node) return node.column && isColumnConst(*node.column) && canBeNativeType(*node.result_type) && node.allow_constant_folding; } -static bool checkIfFunctionIsComparisonEdgeCase(const ActionsDAG::Node & node, const IFunctionBase & impl) -{ - static std::unordered_set comparison_functions - { - NameEquals::name, - NameNotEquals::name, - NameLess::name, - NameGreater::name, - NameLessOrEquals::name, - NameGreaterOrEquals::name - }; - - /** Comparison operator is special case for ActionDAG compilation - * Its result can be constant and we can understand that only during Function execute call. - * It can be a problem if two DAGs with compare function are analyzed, but in first DAG comparison - * function is compiled, in second DAG it is not compiled. - * There will be error because of block headers mismatch. - */ - - auto it = comparison_functions.find(impl.getName()); - if (it == comparison_functions.end()) - return false; - - const auto * lhs_node = node.children[0]; - const auto * rhs_node = node.children[1]; - - while (lhs_node->type == ActionsDAG::ActionType::ALIAS) - lhs_node = lhs_node->children[0]; - - while (rhs_node->type == ActionsDAG::ActionType::ALIAS) - rhs_node = rhs_node->children[0]; - - return lhs_node == rhs_node && !isTuple(lhs_node->result_type); -} - static bool isCompilableFunction(const ActionsDAG::Node & node) { if (node.type != ActionsDAG::ActionType::FUNCTION) @@ -367,9 +332,6 @@ static bool isCompilableFunction(const ActionsDAG::Node & node) return false; } - if (checkIfFunctionIsComparisonEdgeCase(node, *node.function_base)) - return false; - return function.isCompilable(); } From 6c334800c36aa8e5b0b18a6d289f10a1b7806033 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 13 May 2021 17:19:22 +0300 Subject: [PATCH 061/202] Disable broken test --- .../00911_tautological_compare.reference | 8 -------- .../0_stateless/00911_tautological_compare.sql | 16 ++++++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/queries/0_stateless/00911_tautological_compare.reference b/tests/queries/0_stateless/00911_tautological_compare.reference index 405d3348775..e69de29bb2d 100644 --- a/tests/queries/0_stateless/00911_tautological_compare.reference +++ b/tests/queries/0_stateless/00911_tautological_compare.reference @@ -1,8 +0,0 @@ -0 -0 -0 -0 -0 -0 -0 -0 diff --git a/tests/queries/0_stateless/00911_tautological_compare.sql b/tests/queries/0_stateless/00911_tautological_compare.sql index 34c95d73716..5de5fdfda19 100644 --- a/tests/queries/0_stateless/00911_tautological_compare.sql +++ b/tests/queries/0_stateless/00911_tautological_compare.sql @@ -1,10 +1,10 @@ -SELECT count() FROM system.numbers WHERE number != number; -SELECT count() FROM system.numbers WHERE number < number; -SELECT count() FROM system.numbers WHERE number > number; +-- SELECT count() FROM system.numbers WHERE number != number; +-- SELECT count() FROM system.numbers WHERE number < number; +-- SELECT count() FROM system.numbers WHERE number > number; -SELECT count() FROM system.numbers WHERE NOT (number = number); -SELECT count() FROM system.numbers WHERE NOT (number <= number); -SELECT count() FROM system.numbers WHERE NOT (number >= number); +-- SELECT count() FROM system.numbers WHERE NOT (number = number); +-- SELECT count() FROM system.numbers WHERE NOT (number <= number); +-- SELECT count() FROM system.numbers WHERE NOT (number >= number); -SELECT count() FROM system.numbers WHERE SHA256(toString(number)) != SHA256(toString(number)); -SELECT count() FROM system.numbers WHERE SHA256(toString(number)) != SHA256(toString(number)) AND rand() > 10; +-- SELECT count() FROM system.numbers WHERE SHA256(toString(number)) != SHA256(toString(number)); +-- SELECT count() FROM system.numbers WHERE SHA256(toString(number)) != SHA256(toString(number)) AND rand() > 10; From a44ddd2064a2d394cde72ff591009adb82f76489 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 15 May 2021 15:03:13 +0300 Subject: [PATCH 062/202] Update distributed_group_by_no_merge description --- docs/en/operations/settings/settings.md | 4 ++-- src/Core/Settings.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index b0c879af931..1a2f8aba35f 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1520,8 +1520,8 @@ Do not merge aggregation states from different servers for distributed query pro Possible values: - 0 — Disabled (final query processing is done on the initiator node). -- 1 - Do not merge aggregation states from different servers for distributed query processing (query completelly processed on the shard, initiator only proxy the data). -- 2 - Same as 1 but apply `ORDER BY` and `LIMIT` on the initiator (can be used for queries with `ORDER BY` and/or `LIMIT`). +- 1 - Do not merge aggregation states from different servers for distributed query processing (query completelly processed on the shard, initiator only proxy the data), can be used in case it is for certain that there are different keys on different shards. +- 2 - Same as `1` but applies `ORDER BY` and `LIMIT` (it is not possilbe when the query processed completelly on the remote node, like for `distributed_group_by_no_merge=1`) on the initiator (can be used for queries with `ORDER BY` and/or `LIMIT`). **Example** diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 42a20441a2e..75f6c002fff 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -115,7 +115,7 @@ class IColumn; M(Bool, skip_unavailable_shards, false, "If 1, ClickHouse silently skips unavailable shards and nodes unresolvable through DNS. Shard is marked as unavailable when none of the replicas can be reached.", 0) \ \ M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \ - M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed query processing - in case it is for certain that there are different keys on different shards. If 2 - same as 1 but also apply ORDER BY and LIMIT stages", 0) \ + M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \ M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \ M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \ M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \ From 852608c9372d6b11a476abc05262e4d6709569cd Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 15 May 2021 18:01:00 +0300 Subject: [PATCH 063/202] Don't abort on Auth request --- src/Coordination/KeeperStorage.cpp | 13 ++++++++++++- tests/integration/helpers/keeper_config1.xml | 2 ++ tests/integration/helpers/keeper_config2.xml | 2 ++ tests/integration/helpers/keeper_config3.xml | 2 ++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 5f2d6141be9..a449a106576 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -547,6 +547,17 @@ struct KeeperStorageCloseRequest final : public KeeperStorageRequest } }; +/// Dummy implementation TODO: impelement simple ACL +struct KeeperStorageAuthRequest final : public KeeperStorageRequest +{ + using KeeperStorageRequest::KeeperStorageRequest; + std::pair process(KeeperStorage::Container &, KeeperStorage::Ephemerals &, int64_t, int64_t) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); + return { response_ptr, {} }; + } +}; + void KeeperStorage::finalize() { if (finalized) @@ -611,7 +622,7 @@ KeeperWrapperFactory::KeeperWrapperFactory() { registerKeeperRequestWrapper(*this); registerKeeperRequestWrapper(*this); - //registerKeeperRequestWrapper(*this); + registerKeeperRequestWrapper(*this); registerKeeperRequestWrapper(*this); registerKeeperRequestWrapper(*this); registerKeeperRequestWrapper(*this); diff --git a/tests/integration/helpers/keeper_config1.xml b/tests/integration/helpers/keeper_config1.xml index 2d2f66cbc39..687bcff0f54 100644 --- a/tests/integration/helpers/keeper_config1.xml +++ b/tests/integration/helpers/keeper_config1.xml @@ -1,4 +1,6 @@ + true + :: 0.0.0.0 diff --git a/tests/integration/helpers/keeper_config2.xml b/tests/integration/helpers/keeper_config2.xml index 81976015df1..71f4f3552f8 100644 --- a/tests/integration/helpers/keeper_config2.xml +++ b/tests/integration/helpers/keeper_config2.xml @@ -1,4 +1,6 @@ + true + :: 0.0.0.0 diff --git a/tests/integration/helpers/keeper_config3.xml b/tests/integration/helpers/keeper_config3.xml index 07a9e5fc6b7..e235bc448a5 100644 --- a/tests/integration/helpers/keeper_config3.xml +++ b/tests/integration/helpers/keeper_config3.xml @@ -1,4 +1,6 @@ + true + :: 0.0.0.0 From bae419be36f4f913f7307c0a6834dc326e0dbb4f Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 15 May 2021 18:29:07 +0300 Subject: [PATCH 064/202] Fix typo --- src/Coordination/KeeperStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index a449a106576..9e8d2a124e9 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -547,7 +547,7 @@ struct KeeperStorageCloseRequest final : public KeeperStorageRequest } }; -/// Dummy implementation TODO: impelement simple ACL +/// Dummy implementation TODO: implement simple ACL struct KeeperStorageAuthRequest final : public KeeperStorageRequest { using KeeperStorageRequest::KeeperStorageRequest; From e5cd597111de5fcc03119bdf5b780da0425e7d14 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 16 May 2021 16:49:38 +0800 Subject: [PATCH 065/202] Fix add projection to replicated mergetree --- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++++ .../01710_projection_fetch.reference | 8 ++++++++ .../0_stateless/01710_projection_fetch.sql | 20 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 973fc291064..5683109ec2e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4673,6 +4673,10 @@ void StorageReplicatedMergeTree::alter( if (new_indices_str != current_metadata->secondary_indices.toString()) future_metadata_in_zk.skip_indices = new_indices_str; + String new_projections_str = future_metadata.projections.toString(); + if (new_projections_str != current_metadata->projections.toString()) + future_metadata_in_zk.projections = new_projections_str; + String new_constraints_str = future_metadata.constraints.toString(); if (new_constraints_str != current_metadata->constraints.toString()) future_metadata_in_zk.constraints = new_constraints_str; diff --git a/tests/queries/0_stateless/01710_projection_fetch.reference b/tests/queries/0_stateless/01710_projection_fetch.reference index fd20a585633..caa4f5ee511 100644 --- a/tests/queries/0_stateless/01710_projection_fetch.reference +++ b/tests/queries/0_stateless/01710_projection_fetch.reference @@ -9,3 +9,11 @@ 2 2 3 3 4 4 +0 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +2 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +0 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n ),\n PROJECTION pp\n (\n SELECT \n x,\n count()\n GROUP BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 +0 +CREATE TABLE default.tp_2\n(\n `x` Int32,\n `y` Int32,\n PROJECTION p\n (\n SELECT \n x,\n y\n ORDER BY x\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01710_projection_fetch_default\', \'2\')\nORDER BY y\nSETTINGS min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32, index_granularity = 8192 diff --git a/tests/queries/0_stateless/01710_projection_fetch.sql b/tests/queries/0_stateless/01710_projection_fetch.sql index c12dec4cbcc..feb4fe711ea 100644 --- a/tests/queries/0_stateless/01710_projection_fetch.sql +++ b/tests/queries/0_stateless/01710_projection_fetch.sql @@ -15,6 +15,26 @@ insert into tp_1 select number, number from numbers(5); system sync replica tp_2; select * from tp_2 order by x; +-- test projection creation, materialization, clear and drop +alter table tp_1 add projection pp (select x, count() group by x); +system sync replica tp_2; +select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; +show create table tp_2; + +-- all other three operations are mutations +set mutations_sync = 2; +alter table tp_1 materialize projection pp; +select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; +show create table tp_2; + +alter table tp_1 clear projection pp; +select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; +show create table tp_2; + +alter table tp_1 drop projection pp; +select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; +show create table tp_2; + drop table if exists tp_1; drop table if exists tp_2; From f20799f683344c9c1c97ff970b33a7e6b0a81c92 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 12 May 2021 21:14:00 +0300 Subject: [PATCH 066/202] Destroy Buffer() tables before others (within one database) This will reduce amount of data that may be lost at shutdown. v2: replace dynamic_cast with IStorage::isBuffer (@kitaisreal) v3: replace IStorage::isBuffer with IStorage::flush (@alexey-milovidov) v4: flush() for StorageProxy/StorageTableFunction --- src/Databases/DatabasesCommon.cpp | 5 +++++ src/Storages/IStorage.h | 4 ++++ src/Storages/StorageBuffer.cpp | 2 +- src/Storages/StorageBuffer.h | 2 +- src/Storages/StorageProxy.h | 1 + src/Storages/StorageTableFunction.h | 7 +++++++ 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 9329b0a4210..5dcd0017a05 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -125,6 +125,11 @@ void DatabaseWithOwnTablesBase::shutdown() tables_snapshot = tables; } + for (const auto & kv : tables_snapshot) + { + kv.second->flush(); + } + for (const auto & kv : tables_snapshot) { auto table_id = kv.second->getStorageID(); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 6e2ea6aa866..7ed71a23e2f 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -451,6 +451,10 @@ public: */ virtual void shutdown() {} + /// Called before shutdown() to flush data to underlying storage + /// (for Buffer) + virtual void flush() {} + /// Asks table to stop executing some action identified by action_type /// If table does not support such type of lock, and empty lock is returned virtual ActionLock getActionLock(StorageActionBlockType /* action_type */) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index a3ad3f3f13f..6688112c9f1 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -674,7 +674,7 @@ void StorageBuffer::startup() } -void StorageBuffer::shutdown() +void StorageBuffer::flush() { if (!flush_handle) return; diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 586b6426407..d9cb22ef446 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -88,7 +88,7 @@ public: void startup() override; /// Flush all buffers into the subordinate table and stop background thread. - void shutdown() override; + void flush() override; bool optimize( const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 205440261b8..60e7bf24046 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -133,6 +133,7 @@ public: void startup() override { getNested()->startup(); } void shutdown() override { getNested()->shutdown(); } + void flush() override { getNested()->flush(); } ActionLock getActionLock(StorageActionBlockType action_type) override { return getNested()->getActionLock(action_type); } diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 7d909165d5f..e6f21c44fc3 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -62,6 +62,13 @@ public: nested->shutdown(); } + void flush() override + { + std::lock_guard lock{nested_mutex}; + if (nested) + nested->flush(); + } + void drop() override { std::lock_guard lock{nested_mutex}; From 23981ffd0e8fbc6f4cfc87a36ad6b7b85b031520 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 13 May 2021 10:03:00 +0300 Subject: [PATCH 067/202] Introduce IStorage::flushAndShutdown() --- src/Databases/DatabaseLazy.cpp | 2 +- src/Databases/DatabaseReplicated.cpp | 2 +- src/Databases/DatabasesCommon.cpp | 2 +- src/Databases/MySQL/DatabaseConnectionMySQL.cpp | 4 ++-- src/Interpreters/InterpreterDropQuery.cpp | 6 +++--- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- src/Interpreters/SystemLog.h | 2 +- src/Storages/IStorage.h | 15 +++++++++++++++ src/Storages/tests/gtest_storage_log.cpp | 2 +- 9 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index d498cb96062..80179de104a 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -203,7 +203,7 @@ void DatabaseLazy::shutdown() for (const auto & kv : tables_snapshot) { if (kv.second.table) - kv.second.table->shutdown(); + kv.second.table->flushAndShutdown(); } std::lock_guard lock(mutex); diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index b7214917ce8..d36fe45f748 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -529,7 +529,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep dropped_tables.push_back(tryGetTableUUID(table_name)); dropped_dictionaries += table->isDictionary(); - table->shutdown(); + table->flushAndShutdown(); DatabaseAtomic::dropTable(getContext(), table_name, true); } else diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 5dcd0017a05..9d79a0dfe96 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -133,7 +133,7 @@ void DatabaseWithOwnTablesBase::shutdown() for (const auto & kv : tables_snapshot) { auto table_id = kv.second->getStorageID(); - kv.second->shutdown(); + kv.second->flushAndShutdown(); if (table_id.hasUUID()) { assert(getDatabaseName() == DatabaseCatalog::TEMPORARY_DATABASE || getUUID() != UUIDHelpers::Nil); diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp index 59a575996ee..5cd59f8a7c8 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.cpp +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.cpp @@ -316,7 +316,7 @@ void DatabaseConnectionMySQL::shutdown() } for (const auto & [table_name, modify_time_and_storage] : tables_snapshot) - modify_time_and_storage.second->shutdown(); + modify_time_and_storage.second->flushAndShutdown(); std::lock_guard lock(mutex); local_tables_cache.clear(); @@ -343,7 +343,7 @@ void DatabaseConnectionMySQL::cleanOutdatedTables() { const auto table_lock = (*iterator)->lockExclusively(RWLockImpl::NO_QUERY, lock_acquire_timeout); - (*iterator)->shutdown(); + (*iterator)->flushAndShutdown(); (*iterator)->is_dropped = true; iterator = outdated_tables.erase(iterator); } diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 0b4efe4f978..8a9da0cbb0a 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -171,7 +171,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP else table->checkTableCanBeDetached(); - table->shutdown(); + table->flushAndShutdown(); TableExclusiveLockHolder table_lock; if (database->getUUID() == UUIDHelpers::Nil) @@ -215,7 +215,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabaseP else table->checkTableCanBeDropped(); - table->shutdown(); + table->flushAndShutdown(); TableExclusiveLockHolder table_lock; if (database->getUUID() == UUIDHelpers::Nil) @@ -253,7 +253,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, else if (kind == ASTDropQuery::Kind::Drop) { context_handle->removeExternalTable(table_name); - table->shutdown(); + table->flushAndShutdown(); auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); /// Delete table data table->drop(); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 8a4b3b07692..e7babb9b83f 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -436,7 +436,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, if (!table || !dynamic_cast(table.get())) return nullptr; - table->shutdown(); + table->flushAndShutdown(); { /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index aa01ca3517b..074586efe3d 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -153,7 +153,7 @@ public: { stopFlushThread(); if (table) - table->shutdown(); + table->flushAndShutdown(); } String getName() override diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 7ed71a23e2f..a96748ceb63 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -444,6 +444,21 @@ public: */ virtual void startup() {} + /** + * If the storage requires some complicated work on destroying, + * then you have two virtual methods: + * - flush() + * - shutdown() + * + * @see shutdown() + * @see flush() + */ + void flushAndShutdown() + { + flush(); + shutdown(); + } + /** If the table have to do some complicated work when destroying an object - do it in advance. * For example, if the table contains any threads for background work - ask them to complete and wait for completion. * By default, does nothing. diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index e18e4b40ac9..e8057c54a0f 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -53,7 +53,7 @@ public: void TearDown() override { - table->shutdown(); + table->flushAndShutdown(); destroyDisk(disk); } From 5a8d61954a6884b1c403933338850b340d226d5f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 13 May 2021 22:24:20 +0300 Subject: [PATCH 068/202] Call IStorage::flush() for each table on DETACH DATABASE before shutdown --- src/Interpreters/InterpreterDropQuery.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 8a9da0cbb0a..24cff90caea 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -328,6 +328,14 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, query_for_table.database = database_name; query_for_table.no_delay = query.no_delay; + /// Flush should not be done if shouldBeEmptyOnDetach() == false, + /// since in this case getTablesIterator() may do some additional work, + /// see DatabaseMaterializeMySQL<>::getTablesIterator() + for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) + { + iterator->table()->flush(); + } + for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) { DatabasePtr db; From d768a2ac71905474653b8a9cb6ff778641e4f975 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 13 May 2021 22:35:01 +0300 Subject: [PATCH 069/202] Add a test for Buffer flush before shutdown --- .../0_stateless/01870_buffer_flush.reference | 2 ++ .../0_stateless/01870_buffer_flush.sql | 26 +++++++++++++++++++ tests/queries/skip_list.json | 3 ++- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01870_buffer_flush.reference create mode 100644 tests/queries/0_stateless/01870_buffer_flush.sql diff --git a/tests/queries/0_stateless/01870_buffer_flush.reference b/tests/queries/0_stateless/01870_buffer_flush.reference new file mode 100644 index 00000000000..fd3306644d5 --- /dev/null +++ b/tests/queries/0_stateless/01870_buffer_flush.reference @@ -0,0 +1,2 @@ +0 +5 diff --git a/tests/queries/0_stateless/01870_buffer_flush.sql b/tests/queries/0_stateless/01870_buffer_flush.sql new file mode 100644 index 00000000000..55ec79325dc --- /dev/null +++ b/tests/queries/0_stateless/01870_buffer_flush.sql @@ -0,0 +1,26 @@ +-- Check that Buffer will be flushed before shutdown +-- (via DETACH DATABASE) + +drop database if exists db_01870; +create database db_01870; + +-- Right now the order for shutdown is defined and it is: +-- (prefixes are important, to define the order) +-- - a_data_01870 +-- - z_buffer_01870 +-- so on DETACH DATABASE the following error will be printed: +-- +-- Destination table default.a_data_01870 doesn't exist. Block of data is discarded. +create table db_01870.a_data_01870 as system.numbers Engine=TinyLog(); +create table db_01870.z_buffer_01870 as system.numbers Engine=Buffer(db_01870, a_data_01870, 1, + 100, 100, /* time */ + 100, 100, /* rows */ + 100, 1e6 /* bytes */ +); +insert into db_01870.z_buffer_01870 select * from system.numbers limit 5; +select count() from db_01870.a_data_01870; +detach database db_01870; +attach database db_01870; +select count() from db_01870.a_data_01870; + +drop database db_01870; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 0a7acc08664..e29f9efde68 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -720,6 +720,7 @@ "01804_dictionary_decimal256_type", "01850_dist_INSERT_preserve_error", // uses cluster with different static databases shard_0/shard_1 "01821_table_comment", - "01710_projection_fetch" + "01710_projection_fetch", + "01870_buffer_flush" // creates database ] } From 2057b983226abe178c436252cc6a0ff039d0ecf4 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Sun, 16 May 2021 13:48:07 +0300 Subject: [PATCH 070/202] Update 01710_projection_fetch.sql --- tests/queries/0_stateless/01710_projection_fetch.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01710_projection_fetch.sql b/tests/queries/0_stateless/01710_projection_fetch.sql index feb4fe711ea..f5648f44b54 100644 --- a/tests/queries/0_stateless/01710_projection_fetch.sql +++ b/tests/queries/0_stateless/01710_projection_fetch.sql @@ -32,9 +32,9 @@ select count() from system.projection_parts where table = 'tp_2' and name = 'pp' show create table tp_2; alter table tp_1 drop projection pp; -select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; +select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; +system sync replica tp_2; show create table tp_2; drop table if exists tp_1; drop table if exists tp_2; - From 5116f79441f91db5d9021d659c100f8f079918ab Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 16 May 2021 17:18:21 +0300 Subject: [PATCH 071/202] Better configs and fix test --- tests/integration/helpers/cluster.py | 11 ++++++----- tests/integration/helpers/keeper_config1.xml | 1 + tests/integration/helpers/keeper_config2.xml | 1 + tests/integration/helpers/keeper_config3.xml | 1 + tests/integration/test_zookeeper_config/test.py | 6 ++++-- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 1edee6216a6..983a528d7fc 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -37,8 +37,6 @@ DEFAULT_ENV_NAME = 'env_file' SANITIZER_SIGN = "==================" -USE_KEEPER = True - def _create_env_file(path, variables, fname=DEFAULT_ENV_NAME): full_path = os.path.join(path, fname) with open(full_path, 'w') as f: @@ -200,6 +198,7 @@ class ClickHouseCluster: self.schema_registry_port = 8081 self.zookeeper_use_tmpfs = True + self.use_keeper = True self.docker_client = None self.is_up = False @@ -220,7 +219,7 @@ class ClickHouseCluster: with_redis=False, with_minio=False, with_cassandra=False, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, - zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None): + zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True): """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -241,6 +240,8 @@ class ClickHouseCluster: if not env_variables: env_variables = {} + self.use_keeper = use_keeper + # Code coverage files will be placed in database directory # (affect only WITH_COVERAGE=1 build) env_variables['LLVM_PROFILE_FILE'] = '/var/lib/clickhouse/server_%h_%p_%m.profraw' @@ -293,7 +294,7 @@ class ClickHouseCluster: cmds = [] if with_zookeeper and not self.with_zookeeper: if not zookeeper_docker_compose_path: - if USE_KEEPER: + if self.use_keeper: zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_keeper.yml') else: zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_zookeeper.yml') @@ -686,7 +687,7 @@ class ClickHouseCluster: common_opts = ['up', '-d'] if self.with_zookeeper and self.base_zookeeper_cmd: - if USE_KEEPER: + if self.use_keeper: print('Setup Keeper') binary_path = self.server_bin_path if binary_path.endswith('-server'): diff --git a/tests/integration/helpers/keeper_config1.xml b/tests/integration/helpers/keeper_config1.xml index 687bcff0f54..d80f30ebd42 100644 --- a/tests/integration/helpers/keeper_config1.xml +++ b/tests/integration/helpers/keeper_config1.xml @@ -16,6 +16,7 @@ 10000 30000 + trace false diff --git a/tests/integration/helpers/keeper_config2.xml b/tests/integration/helpers/keeper_config2.xml index 71f4f3552f8..8a125cecdb3 100644 --- a/tests/integration/helpers/keeper_config2.xml +++ b/tests/integration/helpers/keeper_config2.xml @@ -16,6 +16,7 @@ 10000 30000 + trace false diff --git a/tests/integration/helpers/keeper_config3.xml b/tests/integration/helpers/keeper_config3.xml index e235bc448a5..04b41677039 100644 --- a/tests/integration/helpers/keeper_config3.xml +++ b/tests/integration/helpers/keeper_config3.xml @@ -16,6 +16,7 @@ 10000 30000 + trace false diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 584f76c80f0..ea8341aebde 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -100,11 +100,12 @@ def test_identity(): cluster_1 = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_with_password.xml') cluster_2 = ClickHouseCluster(__file__) + # TODO ACL not implemented in Keeper. node1 = cluster_1.add_instance('node1', main_configs=["configs/remote_servers.xml", "configs/zookeeper_config_with_password.xml"], - with_zookeeper=True, zookeeper_use_tmpfs=False) + with_zookeeper=True, zookeeper_use_tmpfs=False, use_keeper=False) node2 = cluster_2.add_instance('node2', main_configs=["configs/remote_servers.xml"], with_zookeeper=True, - zookeeper_use_tmpfs=False) + zookeeper_use_tmpfs=False, use_keeper=False) try: cluster_1.start() @@ -126,6 +127,7 @@ def test_identity(): cluster_2.shutdown() +# NOTE this test have to be ported to Keeper def test_secure_connection(): # We need absolute path in zookeeper volumes. Generate it dynamically. TEMPLATE = ''' From e524d11222132df0b30c9b67ecf9f7a0e3e345cd Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 16 May 2021 20:16:10 +0000 Subject: [PATCH 072/202] Remove setting --- src/Storages/RabbitMQ/RabbitMQSettings.h | 1 - src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 23 ++++++++--------------- src/Storages/RabbitMQ/StorageRabbitMQ.h | 1 - 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.h b/src/Storages/RabbitMQ/RabbitMQSettings.h index 185ee2a6eb2..ff7c79b89c6 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.h +++ b/src/Storages/RabbitMQ/RabbitMQSettings.h @@ -19,7 +19,6 @@ namespace DB M(UInt64, rabbitmq_num_consumers, 1, "The number of consumer channels per table.", 0) \ M(UInt64, rabbitmq_num_queues, 1, "The number of queues per consumer.", 0) \ M(String, rabbitmq_queue_base, "", "Base for queue names to be able to reopen non-empty queues in case of failure.", 0) \ - M(String, rabbitmq_deadletter_exchange, "", "Exchange name to be passed as a dead-letter-exchange name.", 0) \ M(Bool, rabbitmq_persistent, false, "For insert query messages will be made 'persistent', durable.", 0) \ M(UInt64, rabbitmq_skip_broken_messages, 0, "Skip at least this number of broken messages from RabbitMQ per block", 0) \ M(UInt64, rabbitmq_max_block_size, 0, "Number of row collected before flushing data from RabbitMQ.", 0) \ diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index ccb24d5db90..bbb2a85d37a 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -87,7 +87,6 @@ StorageRabbitMQ::StorageRabbitMQ( , num_queues(rabbitmq_settings->rabbitmq_num_queues.value) , queue_base(rabbitmq_settings->rabbitmq_queue_base.value) , queue_settings_list(parseSettings(rabbitmq_settings->rabbitmq_queue_settings_list.value)) - , deadletter_exchange(rabbitmq_settings->rabbitmq_deadletter_exchange.value) , persistent(rabbitmq_settings->rabbitmq_persistent.value) , use_user_setup(rabbitmq_settings->rabbitmq_queue_consume.value) , hash_exchange(num_consumers > 1 || num_queues > 1) @@ -487,13 +486,8 @@ void StorageRabbitMQ::bindQueue(size_t queue_id, AMQP::TcpChannel & rabbit_chann { queue_settings["x-max-length"] = queue_size; } - if (!queue_settings.contains("x-dead-letter-exchange") && !deadletter_exchange.empty()) + if (!queue_settings.contains("x-overflow")) { - queue_settings["x-dead-letter-exchange"] = deadletter_exchange; - } - else if (!queue_settings.contains("x-overflow")) - { - /// Define x-overflow only if there is not x-dead-letter-exchange, because it will overwrite the expected behaviour. queue_settings["x-overflow"] = "reject-publish"; } @@ -1068,14 +1062,13 @@ void registerStorageRabbitMQ(StorageFactory & factory) CHECK_RABBITMQ_STORAGE_ARGUMENT(8, rabbitmq_num_consumers) CHECK_RABBITMQ_STORAGE_ARGUMENT(9, rabbitmq_num_queues) CHECK_RABBITMQ_STORAGE_ARGUMENT(10, rabbitmq_queue_base) - CHECK_RABBITMQ_STORAGE_ARGUMENT(11, rabbitmq_deadletter_exchange) - CHECK_RABBITMQ_STORAGE_ARGUMENT(12, rabbitmq_persistent) - CHECK_RABBITMQ_STORAGE_ARGUMENT(13, rabbitmq_skip_broken_messages) - CHECK_RABBITMQ_STORAGE_ARGUMENT(14, rabbitmq_max_block_size) - CHECK_RABBITMQ_STORAGE_ARGUMENT(15, rabbitmq_flush_interval_ms) - CHECK_RABBITMQ_STORAGE_ARGUMENT(16, rabbitmq_vhost) - CHECK_RABBITMQ_STORAGE_ARGUMENT(17, rabbitmq_queue_settings_list) - CHECK_RABBITMQ_STORAGE_ARGUMENT(18, rabbitmq_queue_consume) + CHECK_RABBITMQ_STORAGE_ARGUMENT(11, rabbitmq_persistent) + CHECK_RABBITMQ_STORAGE_ARGUMENT(12, rabbitmq_skip_broken_messages) + CHECK_RABBITMQ_STORAGE_ARGUMENT(13, rabbitmq_max_block_size) + CHECK_RABBITMQ_STORAGE_ARGUMENT(14, rabbitmq_flush_interval_ms) + CHECK_RABBITMQ_STORAGE_ARGUMENT(15, rabbitmq_vhost) + CHECK_RABBITMQ_STORAGE_ARGUMENT(16, rabbitmq_queue_settings_list) + CHECK_RABBITMQ_STORAGE_ARGUMENT(17, rabbitmq_queue_consume) #undef CHECK_RABBITMQ_STORAGE_ARGUMENT diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index fd2da6bba21..c605b0dc0bd 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -92,7 +92,6 @@ private: size_t num_queues; String queue_base; Names queue_settings_list; - const String deadletter_exchange; /// For insert query. Mark messages as durable. const bool persistent; From 4edd5fbb35f5b138a209d74572fe3ef979a44add Mon Sep 17 00:00:00 2001 From: abel-wang Date: Mon, 17 May 2021 12:44:57 +0800 Subject: [PATCH 073/202] Add thread_name in system.stack_trace --- .../System/StorageSystemStackTrace.cpp | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index a6651aff8be..5f9b140ea0c 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -176,6 +177,7 @@ NamesAndTypesList StorageSystemStackTrace::getNamesAndTypes() { return { + { "thread_name", std::make_shared() }, { "thread_id", std::make_shared() }, { "query_id", std::make_shared() }, { "trace", std::make_shared(std::make_shared()) } @@ -213,6 +215,17 @@ void StorageSystemStackTrace::fillData(MutableColumns & res_columns, ContextPtr, throwFromErrno("Cannot send signal with sigqueue", ErrorCodes::CANNOT_SIGQUEUE); } + std::filesystem::path thread_name_path = it->path(); + thread_name_path.append("comm"); + + String thread_name; + if (std::filesystem::exists(thread_name_path)) + { + ReadBufferFromFile comm(thread_name_path.string()); + readStringUntilEOF(thread_name, comm); + comm.close(); + } + /// Just in case we will wait for pipe with timeout. In case signal didn't get processed. if (wait(100) && sig_value.sival_int == data_ready_num.load(std::memory_order_acquire)) @@ -225,9 +238,10 @@ void StorageSystemStackTrace::fillData(MutableColumns & res_columns, ContextPtr, for (size_t i = stack_trace_offset; i < stack_trace_size; ++i) arr.emplace_back(reinterpret_cast(stack_trace.getFramePointers()[i])); - res_columns[0]->insert(tid); - res_columns[1]->insertData(query_id_data, query_id_size); - res_columns[2]->insert(arr); + res_columns[0]->insert(thread_name); + res_columns[1]->insert(tid); + res_columns[2]->insertData(query_id_data, query_id_size); + res_columns[3]->insert(arr); } else { @@ -235,9 +249,10 @@ void StorageSystemStackTrace::fillData(MutableColumns & res_columns, ContextPtr, /// Cannot obtain a stack trace. But create a record in result nevertheless. - res_columns[0]->insert(tid); - res_columns[1]->insertDefault(); + res_columns[0]->insert(thread_name); + res_columns[1]->insert(tid); res_columns[2]->insertDefault(); + res_columns[3]->insertDefault(); } /// Signed integer overflow is undefined behavior in both C and C++. However, according to From f21df0f1863b659e608411c306a33cc1c628125f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 17 May 2021 09:42:22 +0300 Subject: [PATCH 074/202] Update 01710_projection_fetch.sql --- tests/queries/0_stateless/01710_projection_fetch.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01710_projection_fetch.sql b/tests/queries/0_stateless/01710_projection_fetch.sql index f5648f44b54..daba8536eeb 100644 --- a/tests/queries/0_stateless/01710_projection_fetch.sql +++ b/tests/queries/0_stateless/01710_projection_fetch.sql @@ -32,8 +32,8 @@ select count() from system.projection_parts where table = 'tp_2' and name = 'pp' show create table tp_2; alter table tp_1 drop projection pp; -select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; system sync replica tp_2; +select count() from system.projection_parts where table = 'tp_2' and name = 'pp' and active; show create table tp_2; drop table if exists tp_1; From 1dc415596cd76343bc56142c24e837260bc528bc Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Mon, 17 May 2021 10:03:26 +0300 Subject: [PATCH 075/202] Fix race condition in ContextAccess. --- src/Access/ContextAccess.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 0bcaef1e441..90495a83dfc 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -143,11 +143,13 @@ ContextAccess::ContextAccess(const AccessControlManager & manager_, const Params : manager(&manager_) , params(params_) { + std::lock_guard lock{mutex}; + subscription_for_user_change = manager->subscribeForChanges( *params.user_id, [this](const UUID &, const AccessEntityPtr & entity) { UserPtr changed_user = entity ? typeid_cast(entity) : nullptr; - std::lock_guard lock{mutex}; + std::lock_guard lock2{mutex}; setUser(changed_user); }); @@ -189,7 +191,7 @@ void ContextAccess::setUser(const UserPtr & user_) const current_roles_with_admin_option = user->granted_roles.findGrantedWithAdminOption(params.current_roles); } - subscription_for_roles_changes = {}; + subscription_for_roles_changes.reset(); enabled_roles = manager->getEnabledRoles(current_roles, current_roles_with_admin_option); subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr & roles_info_) { From 3a822d28321e2bcb80739d9b84aa9d53d213ce80 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 17 May 2021 10:25:50 +0300 Subject: [PATCH 076/202] Added comments --- tests/performance/column_column_comparison.xml | 4 +++- tests/queries/0_stateless/00911_tautological_compare.sql | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/performance/column_column_comparison.xml b/tests/performance/column_column_comparison.xml index 2b59a65a54b..15354c86f55 100644 --- a/tests/performance/column_column_comparison.xml +++ b/tests/performance/column_column_comparison.xml @@ -1,3 +1,5 @@ + diff --git a/tests/queries/0_stateless/00911_tautological_compare.sql b/tests/queries/0_stateless/00911_tautological_compare.sql index 5de5fdfda19..b477aabc025 100644 --- a/tests/queries/0_stateless/00911_tautological_compare.sql +++ b/tests/queries/0_stateless/00911_tautological_compare.sql @@ -1,3 +1,4 @@ +-- TODO: Return this test after simplifying InterpreterSelectQuery -- SELECT count() FROM system.numbers WHERE number != number; -- SELECT count() FROM system.numbers WHERE number < number; -- SELECT count() FROM system.numbers WHERE number > number; From d923d9e6efbb710009a046175e1d86db78b4ef08 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 17 May 2021 10:30:42 +0300 Subject: [PATCH 077/202] Function move file --- src/CMakeLists.txt | 4 +- src/Functions/CMakeLists.txt | 4 +- src/Functions/CustomWeekTransforms.h | 2 +- src/Functions/DateTimeTransforms.h | 2 +- src/Functions/FunctionBinaryArithmetic.h | 2 +- src/Functions/FunctionBitTestMany.h | 2 +- src/Functions/FunctionCustomWeekToSomething.h | 2 +- .../FunctionDateOrDateTimeAddInterval.h | 2 +- .../FunctionDateOrDateTimeToSomething.h | 2 +- src/Functions/FunctionFQDN.cpp | 2 +- src/Functions/FunctionFactory.h | 2 +- src/Functions/FunctionHelpers.cpp | 2 +- src/Functions/FunctionIfBase.h | 2 +- src/Functions/FunctionMathBinaryFloat64.h | 2 +- src/Functions/FunctionMathConstFloat64.h | 2 +- src/Functions/FunctionMathUnary.h | 2 +- src/Functions/FunctionNumericPredicate.h | 2 +- src/Functions/FunctionStartsEndsWith.h | 2 +- src/Functions/FunctionStringOrArrayToT.h | 2 +- src/Functions/FunctionStringToString.h | 2 +- src/Functions/FunctionUnaryArithmetic.h | 2 +- src/Functions/FunctionUnixTimestamp64.h | 2 +- src/Functions/FunctionsBitmap.h | 2 +- src/Functions/FunctionsCoding.h | 2 +- src/Functions/FunctionsConsistentHashing.h | 2 +- src/Functions/FunctionsEmbeddedDictionaries.h | 2 +- src/Functions/FunctionsExternalDictionaries.h | 2 +- src/Functions/FunctionsExternalModels.h | 2 +- src/Functions/FunctionsHashing.h | 2 +- src/Functions/FunctionsJSON.h | 2 +- src/Functions/FunctionsLogical.h | 2 +- .../FunctionsMultiStringFuzzySearch.h | 2 +- src/Functions/FunctionsMultiStringPosition.h | 2 +- src/Functions/FunctionsMultiStringSearch.h | 2 +- src/Functions/FunctionsRandom.h | 2 +- src/Functions/FunctionsRound.h | 2 +- src/Functions/FunctionsStringArray.h | 2 +- src/Functions/FunctionsStringHash.h | 2 +- src/Functions/FunctionsStringSearch.h | 2 +- src/Functions/FunctionsStringSearchToString.h | 2 +- src/Functions/FunctionsStringSimilarity.h | 2 +- src/Functions/FunctionsVisitParam.h | 2 +- src/Functions/IFunction.cpp | 84 ++++++++++++ src/Functions/IFunction.h | 116 ++++++++++++++++ src/Functions/IFunctionAdaptors.h | 2 +- src/Functions/IFunctionOld.cpp | 91 ------------ src/Functions/IFunctionOld.h | 129 ------------------ src/Functions/LeastGreatestGeneric.h | 2 +- src/Functions/PerformanceAdaptors.h | 2 +- src/Functions/URL/port.cpp | 2 +- src/Functions/addressToLine.cpp | 2 +- src/Functions/addressToSymbol.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 2 +- src/Functions/array/FunctionArrayMapped.h | 2 +- src/Functions/array/arrayConcat.cpp | 2 +- src/Functions/array/arrayDistinct.cpp | 2 +- src/Functions/array/arrayElement.cpp | 2 +- src/Functions/array/arrayEnumerate.cpp | 2 +- src/Functions/array/arrayEnumerateExtended.h | 2 +- src/Functions/array/arrayEnumerateRanked.h | 2 +- src/Functions/array/arrayFlatten.cpp | 2 +- src/Functions/array/arrayIndex.h | 2 +- src/Functions/array/arrayIntersect.cpp | 2 +- src/Functions/array/arrayJoin.cpp | 2 +- src/Functions/array/arrayPop.h | 2 +- src/Functions/array/arrayPush.h | 2 +- src/Functions/array/arrayReduce.cpp | 2 +- src/Functions/array/arrayReduceInRanges.cpp | 2 +- src/Functions/array/arrayResize.cpp | 2 +- src/Functions/array/arrayReverse.cpp | 2 +- src/Functions/array/arraySlice.cpp | 2 +- src/Functions/array/arrayUniq.cpp | 2 +- src/Functions/array/arrayWithConstant.cpp | 2 +- src/Functions/array/emptyArray.cpp | 2 +- src/Functions/array/emptyArrayToSingle.cpp | 2 +- src/Functions/array/hasAllAny.h | 2 +- src/Functions/array/range.cpp | 2 +- src/Functions/assumeNotNull.cpp | 2 +- src/Functions/bar.cpp | 2 +- src/Functions/bitmaskToList.cpp | 2 +- src/Functions/blockNumber.cpp | 2 +- src/Functions/blockSerializedSize.cpp | 2 +- src/Functions/blockSize.cpp | 2 +- src/Functions/buildId.cpp | 2 +- src/Functions/byteSize.cpp | 2 +- src/Functions/coalesce.cpp | 2 +- src/Functions/concat.cpp | 2 +- src/Functions/connectionId.cpp | 2 +- src/Functions/countDigits.cpp | 2 +- src/Functions/countMatches.h | 2 +- src/Functions/currentDatabase.cpp | 2 +- src/Functions/currentUser.cpp | 2 +- src/Functions/dateDiff.cpp | 2 +- src/Functions/defaultValueOfArgumentType.cpp | 2 +- src/Functions/defaultValueOfTypeName.cpp | 2 +- src/Functions/dumpColumnStructure.cpp | 2 +- src/Functions/errorCodeToName.cpp | 2 +- src/Functions/evalMLMethod.cpp | 2 +- src/Functions/extractAllGroups.h | 2 +- src/Functions/extractTextFromHTML.cpp | 2 +- src/Functions/filesystem.cpp | 2 +- src/Functions/finalizeAggregation.cpp | 2 +- src/Functions/formatDateTime.cpp | 2 +- src/Functions/formatReadable.h | 2 +- src/Functions/formatReadableTimeDelta.cpp | 2 +- src/Functions/formatRow.cpp | 2 +- src/Functions/formatString.cpp | 2 +- src/Functions/fuzzBits.cpp | 2 +- src/Functions/geoToH3.cpp | 2 +- src/Functions/geohashesInBox.cpp | 2 +- src/Functions/getMacro.cpp | 2 +- src/Functions/getScalar.cpp | 2 +- src/Functions/getSetting.cpp | 2 +- src/Functions/getSizeOfEnumType.cpp | 2 +- src/Functions/globalVariable.cpp | 2 +- src/Functions/greatCircleDistance.cpp | 2 +- src/Functions/hasColumnInTable.cpp | 2 +- src/Functions/hostName.cpp | 2 +- src/Functions/identity.cpp | 2 +- src/Functions/if.cpp | 2 +- src/Functions/ifNotFinite.cpp | 2 +- src/Functions/ifNull.cpp | 2 +- src/Functions/ignore.cpp | 2 +- src/Functions/in.cpp | 2 +- src/Functions/indexHint.cpp | 2 +- src/Functions/initializeAggregation.cpp | 2 +- src/Functions/isConstant.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Functions/isIPAddressContainedIn.cpp | 2 +- src/Functions/isNotNull.cpp | 2 +- src/Functions/isNull.cpp | 2 +- src/Functions/isZeroOrNull.cpp | 2 +- src/Functions/logTrace.cpp | 2 +- src/Functions/lowCardinalityIndices.cpp | 2 +- src/Functions/lowCardinalityKeys.cpp | 2 +- src/Functions/map.cpp | 2 +- src/Functions/materialize.h | 2 +- src/Functions/neighbor.cpp | 2 +- src/Functions/nullIf.cpp | 2 +- src/Functions/partitionId.cpp | 2 +- src/Functions/pointInEllipses.cpp | 2 +- src/Functions/randomFixedString.cpp | 2 +- src/Functions/randomPrintableASCII.cpp | 2 +- src/Functions/randomString.cpp | 2 +- src/Functions/randomStringUTF8.cpp | 2 +- src/Functions/repeat.cpp | 2 +- src/Functions/replicate.cpp | 2 +- src/Functions/replicate.h | 2 +- src/Functions/rowNumberInAllBlocks.cpp | 2 +- src/Functions/rowNumberInBlock.cpp | 2 +- src/Functions/runningAccumulate.cpp | 2 +- src/Functions/runningConcurrency.cpp | 2 +- src/Functions/runningDifference.h | 2 +- src/Functions/sleep.h | 2 +- src/Functions/substring.cpp | 2 +- src/Functions/subtractDays.cpp | 2 +- src/Functions/subtractHours.cpp | 2 +- src/Functions/subtractMinutes.cpp | 2 +- src/Functions/subtractMonths.cpp | 2 +- src/Functions/subtractQuarters.cpp | 2 +- src/Functions/subtractSeconds.cpp | 2 +- src/Functions/subtractWeeks.cpp | 2 +- src/Functions/subtractYears.cpp | 2 +- src/Functions/throwIf.cpp | 2 +- src/Functions/tid.cpp | 2 +- src/Functions/timeSlot.cpp | 2 +- src/Functions/timeSlots.cpp | 2 +- src/Functions/timezone.cpp | 2 +- src/Functions/timezoneOf.cpp | 2 +- src/Functions/toColumnTypeName.cpp | 2 +- src/Functions/toCustomWeek.cpp | 2 +- src/Functions/toFixedString.h | 2 +- src/Functions/toLowCardinality.cpp | 2 +- src/Functions/toModifiedJulianDay.cpp | 2 +- src/Functions/toNullable.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toTimezone.cpp | 2 +- src/Functions/toTypeName.cpp | 2 +- src/Functions/today.cpp | 2 +- src/Functions/transform.cpp | 2 +- src/Functions/tuple.cpp | 2 +- src/Functions/tupleElement.cpp | 2 +- src/Functions/uptime.cpp | 2 +- src/Functions/version.cpp | 2 +- src/Functions/visibleWidth.cpp | 2 +- src/Functions/yesterday.cpp | 2 +- src/Interpreters/JIT/CompileDAG.h | 2 +- 187 files changed, 385 insertions(+), 405 deletions(-) delete mode 100644 src/Functions/IFunctionOld.cpp delete mode 100644 src/Functions/IFunctionOld.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d22a69c211c..a0f36163d68 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -106,8 +106,8 @@ endif() list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) -list (APPEND dbms_sources Functions/IFunction.cpp Functions/IFunctionOld.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) -list (APPEND dbms_headers Functions/IFunctionOld.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) +list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) +list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) list (APPEND dbms_sources AggregateFunctions/AggregateFunctionFactory.cpp diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index ba876e02424..271deb0f42c 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -5,8 +5,8 @@ add_subdirectory(divide) include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions .) -list(REMOVE_ITEM clickhouse_functions_sources IFunctionOld.cpp FunctionFactory.cpp FunctionHelpers.cpp) -list(REMOVE_ITEM clickhouse_functions_headers IFunctionOld.h FunctionFactory.h FunctionHelpers.h) +list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp) +list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h) add_library(clickhouse_functions ${clickhouse_functions_sources}) diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index f07f2777cec..d1a8e3a54fd 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index aa8f52b335e..03f35333150 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 0d90eece3e2..927b870891f 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -23,7 +23,7 @@ #include #include #include "Core/DecimalFunctions.h" -#include "IFunctionOld.h" +#include "IFunction.h" #include "FunctionHelpers.h" #include "IsOperation.h" #include "DivisionUtils.h" diff --git a/src/Functions/FunctionBitTestMany.h b/src/Functions/FunctionBitTestMany.h index 6ed787d707e..567be37cda8 100644 --- a/src/Functions/FunctionBitTestMany.h +++ b/src/Functions/FunctionBitTestMany.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index c229764b8a4..0ef6a342218 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index d0b05c83eaa..14ab9f00452 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -7,7 +7,7 @@ #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index ded6c157433..8b8f1b483c8 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionFQDN.cpp b/src/Functions/FunctionFQDN.cpp index 439181862a7..304aad17d22 100644 --- a/src/Functions/FunctionFQDN.cpp +++ b/src/Functions/FunctionFQDN.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index 966ee2df952..dd7d19d8d07 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 8fc21fc0c57..dcdd0e521eb 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionIfBase.h b/src/Functions/FunctionIfBase.h index 32f0f629287..17d02de2e27 100644 --- a/src/Functions/FunctionIfBase.h +++ b/src/Functions/FunctionIfBase.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #if !defined(ARCADIA_BUILD) diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h index 2f43981f751..8cc012d3ab2 100644 --- a/src/Functions/FunctionMathBinaryFloat64.h +++ b/src/Functions/FunctionMathBinaryFloat64.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #if !defined(ARCADIA_BUILD) diff --git a/src/Functions/FunctionMathConstFloat64.h b/src/Functions/FunctionMathConstFloat64.h index f983988a9ae..ab7d401e99e 100644 --- a/src/Functions/FunctionMathConstFloat64.h +++ b/src/Functions/FunctionMathConstFloat64.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace DB diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index 5f310e2d8f8..a637bbe3bd8 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #if !defined(ARCADIA_BUILD) diff --git a/src/Functions/FunctionNumericPredicate.h b/src/Functions/FunctionNumericPredicate.h index d27739c28de..d5fa9f03aba 100644 --- a/src/Functions/FunctionNumericPredicate.h +++ b/src/Functions/FunctionNumericPredicate.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index ead22ab15d1..65dbf393290 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h index 1e163b2ebef..158179fffe9 100644 --- a/src/Functions/FunctionStringOrArrayToT.h +++ b/src/Functions/FunctionStringOrArrayToT.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h index 50341ba149e..26480a83995 100644 --- a/src/Functions/FunctionStringToString.h +++ b/src/Functions/FunctionStringToString.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index ee9cb914831..68fcfadfb84 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index 969b31cd13b..a4357c148ac 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h index 9bdcb2f81cb..bbdc53c3006 100644 --- a/src/Functions/FunctionsBitmap.h +++ b/src/Functions/FunctionsBitmap.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 512afc01674..01602cf1175 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsConsistentHashing.h b/src/Functions/FunctionsConsistentHashing.h index 2448cee3f46..4c393f6ee01 100644 --- a/src/Functions/FunctionsConsistentHashing.h +++ b/src/Functions/FunctionsConsistentHashing.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index aa32521ba6a..884f53125eb 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index e2ee7dd31f3..6d9e4ce6768 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -27,7 +27,7 @@ #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsExternalModels.h b/src/Functions/FunctionsExternalModels.h index f3e727cfcef..8f8b0e0c860 100644 --- a/src/Functions/FunctionsExternalModels.h +++ b/src/Functions/FunctionsExternalModels.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace DB diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index efd9a46b1d8..690991759a3 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index ebc828fb287..2b49d01de2c 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/FunctionsLogical.h b/src/Functions/FunctionsLogical.h index 94bbafc5808..acc34a40d00 100644 --- a/src/Functions/FunctionsLogical.h +++ b/src/Functions/FunctionsLogical.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h index 143fa445a1c..209efb0fc2f 100644 --- a/src/Functions/FunctionsMultiStringFuzzySearch.h +++ b/src/Functions/FunctionsMultiStringFuzzySearch.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsMultiStringPosition.h b/src/Functions/FunctionsMultiStringPosition.h index e68a5f87f35..f36f7639ccd 100644 --- a/src/Functions/FunctionsMultiStringPosition.h +++ b/src/Functions/FunctionsMultiStringPosition.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h index 43f9c89e333..08b4668940e 100644 --- a/src/Functions/FunctionsMultiStringSearch.h +++ b/src/Functions/FunctionsMultiStringSearch.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsRandom.h b/src/Functions/FunctionsRandom.h index 08b58e8a1a7..75037d02a2d 100644 --- a/src/Functions/FunctionsRandom.h +++ b/src/Functions/FunctionsRandom.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsRound.h b/src/Functions/FunctionsRound.h index a56f609e2f8..f56b92d6db5 100644 --- a/src/Functions/FunctionsRound.h +++ b/src/Functions/FunctionsRound.h @@ -9,7 +9,7 @@ #include #include #include -#include "IFunctionOld.h" +#include "IFunction.h" #include #include #include diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h index 046bfbfb6e9..27f10797651 100644 --- a/src/Functions/FunctionsStringArray.h +++ b/src/Functions/FunctionsStringArray.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsStringHash.h b/src/Functions/FunctionsStringHash.h index 00f38117dfa..37fa7d618b9 100644 --- a/src/Functions/FunctionsStringHash.h +++ b/src/Functions/FunctionsStringHash.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include namespace DB diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h index 489332833c6..0789247e2d4 100644 --- a/src/Functions/FunctionsStringSearch.h +++ b/src/Functions/FunctionsStringSearch.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsStringSearchToString.h b/src/Functions/FunctionsStringSearchToString.h index 92abd017c23..af91a9511e1 100644 --- a/src/Functions/FunctionsStringSearchToString.h +++ b/src/Functions/FunctionsStringSearchToString.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/FunctionsStringSimilarity.h b/src/Functions/FunctionsStringSimilarity.h index fcc500cd66f..6efb373aace 100644 --- a/src/Functions/FunctionsStringSimilarity.h +++ b/src/Functions/FunctionsStringSimilarity.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include namespace DB diff --git a/src/Functions/FunctionsVisitParam.h b/src/Functions/FunctionsVisitParam.h index 61b361c97fa..922ea44be9a 100644 --- a/src/Functions/FunctionsVisitParam.h +++ b/src/Functions/FunctionsVisitParam.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 6fa3cb130e5..eada0641f2d 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -17,6 +17,17 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if USE_EMBEDDED_COMPILER +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-parameter" +# include +# pragma GCC diagnostic pop +#endif + namespace DB { @@ -343,4 +354,77 @@ DataTypePtr IFunctionOverloadResolver::getReturnTypeWithoutLowCardinality(const return getReturnTypeImpl(arguments); } + +#if USE_EMBEDDED_COMPILER + +static std::optional removeNullables(const DataTypes & types) +{ + for (const auto & type : types) + { + if (!typeid_cast(type.get())) + continue; + DataTypes filtered; + for (const auto & sub_type : types) + filtered.emplace_back(removeNullable(sub_type)); + return filtered; + } + return {}; +} + +bool IFunction::isCompilable(const DataTypes & arguments) const +{ + if (useDefaultImplementationForNulls()) + if (auto denulled = removeNullables(arguments)) + return isCompilableImpl(*denulled); + return isCompilableImpl(arguments); +} + +llvm::Value * IFunction::compile(llvm::IRBuilderBase & builder, const DataTypes & arguments, Values values) const +{ + auto denulled_arguments = removeNullables(arguments); + if (useDefaultImplementationForNulls() && denulled_arguments) + { + auto & b = static_cast &>(builder); + + std::vector unwrapped_values; + std::vector is_null_values; + + unwrapped_values.reserve(arguments.size()); + is_null_values.reserve(arguments.size()); + + for (size_t i = 0; i < arguments.size(); ++i) + { + auto * value = values[i]; + + WhichDataType data_type(arguments[i]); + if (data_type.isNullable()) + { + unwrapped_values.emplace_back(b.CreateExtractValue(value, {0})); + is_null_values.emplace_back(b.CreateExtractValue(value, {1})); + } + else + { + unwrapped_values.emplace_back(value); + } + } + + auto * result = compileImpl(builder, *denulled_arguments, unwrapped_values); + + auto * nullable_structure_type = toNativeType(b, makeNullable(getReturnTypeImpl(*denulled_arguments))); + auto * nullable_structure_value = llvm::Constant::getNullValue(nullable_structure_type); + + auto * nullable_structure_with_result_value = b.CreateInsertValue(nullable_structure_value, result, {0}); + auto * nullable_structure_result_null = b.CreateExtractValue(nullable_structure_with_result_value, {1}); + + for (auto * is_null_value : is_null_values) + nullable_structure_result_null = b.CreateOr(nullable_structure_result_null, is_null_value); + + return b.CreateInsertValue(nullable_structure_with_result_value, nullable_structure_result_null, {1}); + } + + return compileImpl(builder, arguments, std::move(values)); +} + +#endif + } diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 05a77699b82..fe3ec21afa1 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -326,4 +326,120 @@ private: using FunctionOverloadResolverPtr = std::shared_ptr; +/// Old function interface. Check documentation in IFunction.h. +/// If client do not need statefull properties it can implement this interface. +class IFunction +{ +public: + + virtual ~IFunction() = default; + + virtual String getName() const = 0; + + virtual ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const = 0; + virtual ColumnPtr executeImplDryRun(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const + { + return executeImpl(arguments, result_type, input_rows_count); + } + + /** Default implementation in presence of Nullable arguments or NULL constants as arguments is the following: + * if some of arguments are NULL constants then return NULL constant, + * if some of arguments are Nullable, then execute function as usual for columns, + * where Nullable columns are substituted with nested columns (they have arbitrary values in rows corresponding to NULL value) + * and wrap result in Nullable column where NULLs are in all rows where any of arguments are NULL. + */ + virtual bool useDefaultImplementationForNulls() const { return true; } + + /** If the function have non-zero number of arguments, + * and if all arguments are constant, that we could automatically provide default implementation: + * arguments are converted to ordinary columns with single value, then function is executed as usual, + * and then the result is converted to constant column. + */ + virtual bool useDefaultImplementationForConstants() const { return false; } + + /** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column. + * Otherwise, convert all low cardinality columns to ordinary columns. + * Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality. + */ + virtual bool useDefaultImplementationForLowCardinalityColumns() const { return true; } + + /// If it isn't, will convert all ColumnLowCardinality arguments to full columns. + virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; } + + /** Some arguments could remain constant during this implementation. + */ + virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; } + + /** True if function can be called on default arguments (include Nullable's) and won't throw. + * Counterexample: modulo(0, 0) + */ + virtual bool canBeExecutedOnDefaultArguments() const { return true; } + + /// Properties from IFunctionBase (see IFunction.h) + virtual bool isSuitableForConstantFolding() const { return true; } + virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; } + virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; } + virtual bool isDeterministic() const { return true; } + virtual bool isDeterministicInScopeOfQuery() const { return true; } + virtual bool isStateful() const { return false; } + virtual bool hasInformationAboutMonotonicity() const { return false; } + + using Monotonicity = IFunctionBase::Monotonicity; + virtual Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const + { + throw Exception("Function " + getName() + " has no information about its monotonicity.", ErrorCodes::NOT_IMPLEMENTED); + } + + /// For non-variadic functions, return number of arguments; otherwise return zero (that should be ignored). + virtual size_t getNumberOfArguments() const = 0; + + virtual DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const + { + throw Exception("getReturnType is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + + /// Get the result type by argument type. If the function does not apply to these arguments, throw an exception. + virtual DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const + { + DataTypes data_types(arguments.size()); + for (size_t i = 0; i < arguments.size(); ++i) + data_types[i] = arguments[i].type; + + return getReturnTypeImpl(data_types); + } + + virtual bool isVariadic() const { return false; } + + virtual void getLambdaArgumentTypes(DataTypes & /*arguments*/) const + { + throw Exception("Function " + getName() + " can't have lambda-expressions as arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + virtual ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const { return {}; } + + +#if USE_EMBEDDED_COMPILER + + bool isCompilable(const DataTypes & arguments) const; + + llvm::Value * compile(llvm::IRBuilderBase &, const DataTypes & arguments, Values values) const; + +#endif + +protected: + +#if USE_EMBEDDED_COMPILER + + virtual bool isCompilableImpl(const DataTypes &) const { return false; } + + virtual llvm::Value * compileImpl(llvm::IRBuilderBase &, const DataTypes &, Values) const + { + throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED); + } + +#endif +}; + +using FunctionPtr = std::shared_ptr; + } diff --git a/src/Functions/IFunctionAdaptors.h b/src/Functions/IFunctionAdaptors.h index 313d006e00c..b213a459103 100644 --- a/src/Functions/IFunctionAdaptors.h +++ b/src/Functions/IFunctionAdaptors.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Functions/IFunctionOld.cpp b/src/Functions/IFunctionOld.cpp deleted file mode 100644 index c25247a9742..00000000000 --- a/src/Functions/IFunctionOld.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include "IFunctionOld.h" - -#include - -#if !defined(ARCADIA_BUILD) -# include -#endif - -#if USE_EMBEDDED_COMPILER -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-parameter" -# include -# pragma GCC diagnostic pop -#endif - -namespace DB -{ - -#if USE_EMBEDDED_COMPILER - -static std::optional removeNullables(const DataTypes & types) -{ - for (const auto & type : types) - { - if (!typeid_cast(type.get())) - continue; - DataTypes filtered; - for (const auto & sub_type : types) - filtered.emplace_back(removeNullable(sub_type)); - return filtered; - } - return {}; -} - -bool IFunction::isCompilable(const DataTypes & arguments) const -{ - if (useDefaultImplementationForNulls()) - if (auto denulled = removeNullables(arguments)) - return isCompilableImpl(*denulled); - return isCompilableImpl(arguments); -} - -llvm::Value * IFunction::compile(llvm::IRBuilderBase & builder, const DataTypes & arguments, Values values) const -{ - auto denulled_arguments = removeNullables(arguments); - if (useDefaultImplementationForNulls() && denulled_arguments) - { - auto & b = static_cast &>(builder); - - std::vector unwrapped_values; - std::vector is_null_values; - - unwrapped_values.reserve(arguments.size()); - is_null_values.reserve(arguments.size()); - - for (size_t i = 0; i < arguments.size(); ++i) - { - auto * value = values[i]; - - WhichDataType data_type(arguments[i]); - if (data_type.isNullable()) - { - unwrapped_values.emplace_back(b.CreateExtractValue(value, {0})); - is_null_values.emplace_back(b.CreateExtractValue(value, {1})); - } - else - { - unwrapped_values.emplace_back(value); - } - } - - auto * result = compileImpl(builder, *denulled_arguments, unwrapped_values); - - auto * nullable_structure_type = toNativeType(b, makeNullable(getReturnTypeImpl(*denulled_arguments))); - auto * nullable_structure_value = llvm::Constant::getNullValue(nullable_structure_type); - - auto * nullable_structure_with_result_value = b.CreateInsertValue(nullable_structure_value, result, {0}); - auto * nullable_structure_result_null = b.CreateExtractValue(nullable_structure_with_result_value, {1}); - - for (auto * is_null_value : is_null_values) - nullable_structure_result_null = b.CreateOr(nullable_structure_result_null, is_null_value); - - return b.CreateInsertValue(nullable_structure_with_result_value, nullable_structure_result_null, {1}); - } - - return compileImpl(builder, arguments, std::move(values)); -} - -#endif - -} diff --git a/src/Functions/IFunctionOld.h b/src/Functions/IFunctionOld.h deleted file mode 100644 index 4b8ef64bc65..00000000000 --- a/src/Functions/IFunctionOld.h +++ /dev/null @@ -1,129 +0,0 @@ -#pragma once - -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NOT_IMPLEMENTED; -} - -/// Old function interface. Check documentation in IFunction.h -class IFunction -{ -public: - - virtual ~IFunction() = default; - - virtual String getName() const = 0; - - virtual ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const = 0; - virtual ColumnPtr executeImplDryRun(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const - { - return executeImpl(arguments, result_type, input_rows_count); - } - - /** Default implementation in presence of Nullable arguments or NULL constants as arguments is the following: - * if some of arguments are NULL constants then return NULL constant, - * if some of arguments are Nullable, then execute function as usual for columns, - * where Nullable columns are substituted with nested columns (they have arbitrary values in rows corresponding to NULL value) - * and wrap result in Nullable column where NULLs are in all rows where any of arguments are NULL. - */ - virtual bool useDefaultImplementationForNulls() const { return true; } - - /** If the function have non-zero number of arguments, - * and if all arguments are constant, that we could automatically provide default implementation: - * arguments are converted to ordinary columns with single value, then function is executed as usual, - * and then the result is converted to constant column. - */ - virtual bool useDefaultImplementationForConstants() const { return false; } - - /** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column. - * Otherwise, convert all low cardinality columns to ordinary columns. - * Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality. - */ - virtual bool useDefaultImplementationForLowCardinalityColumns() const { return true; } - - /// If it isn't, will convert all ColumnLowCardinality arguments to full columns. - virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; } - - /** Some arguments could remain constant during this implementation. - */ - virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; } - - /** True if function can be called on default arguments (include Nullable's) and won't throw. - * Counterexample: modulo(0, 0) - */ - virtual bool canBeExecutedOnDefaultArguments() const { return true; } - - /// Properties from IFunctionBase (see IFunction.h) - virtual bool isSuitableForConstantFolding() const { return true; } - virtual ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const ColumnsWithTypeAndName & /*arguments*/) const { return nullptr; } - virtual bool isInjective(const ColumnsWithTypeAndName & /*sample_columns*/) const { return false; } - virtual bool isDeterministic() const { return true; } - virtual bool isDeterministicInScopeOfQuery() const { return true; } - virtual bool isStateful() const { return false; } - virtual bool hasInformationAboutMonotonicity() const { return false; } - - using Monotonicity = IFunctionBase::Monotonicity; - virtual Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const - { - throw Exception("Function " + getName() + " has no information about its monotonicity.", ErrorCodes::NOT_IMPLEMENTED); - } - - /// For non-variadic functions, return number of arguments; otherwise return zero (that should be ignored). - virtual size_t getNumberOfArguments() const = 0; - - virtual DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const - { - throw Exception("getReturnType is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } - - /// Get the result type by argument type. If the function does not apply to these arguments, throw an exception. - virtual DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const - { - DataTypes data_types(arguments.size()); - for (size_t i = 0; i < arguments.size(); ++i) - data_types[i] = arguments[i].type; - - return getReturnTypeImpl(data_types); - } - - virtual bool isVariadic() const { return false; } - - virtual void getLambdaArgumentTypes(DataTypes & /*arguments*/) const - { - throw Exception("Function " + getName() + " can't have lambda-expressions as arguments", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - virtual ColumnNumbers getArgumentsThatDontImplyNullableReturnType(size_t /*number_of_arguments*/) const { return {}; } - - -#if USE_EMBEDDED_COMPILER - - bool isCompilable(const DataTypes & arguments) const; - - llvm::Value * compile(llvm::IRBuilderBase &, const DataTypes & arguments, Values values) const; - -#endif - -protected: - -#if USE_EMBEDDED_COMPILER - - virtual bool isCompilableImpl(const DataTypes &) const { return false; } - - virtual llvm::Value * compileImpl(llvm::IRBuilderBase &, const DataTypes &, Values) const - { - throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED); - } - -#endif -}; - -using FunctionPtr = std::shared_ptr; - -} diff --git a/src/Functions/LeastGreatestGeneric.h b/src/Functions/LeastGreatestGeneric.h index bc36372174a..51564ad7cdc 100644 --- a/src/Functions/LeastGreatestGeneric.h +++ b/src/Functions/LeastGreatestGeneric.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/PerformanceAdaptors.h b/src/Functions/PerformanceAdaptors.h index 18fe31ec762..9ef6454d085 100644 --- a/src/Functions/PerformanceAdaptors.h +++ b/src/Functions/PerformanceAdaptors.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp index 46197f43c8c..179a2be4471 100644 --- a/src/Functions/URL/port.cpp +++ b/src/Functions/URL/port.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index 5c83db50f08..aabf388b428 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/addressToSymbol.cpp b/src/Functions/addressToSymbol.cpp index fd1664288ed..7ffdc6d4260 100644 --- a/src/Functions/addressToSymbol.cpp +++ b/src/Functions/addressToSymbol.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 359d20c9615..266b2a44f4a 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/array/FunctionArrayMapped.h b/src/Functions/array/FunctionArrayMapped.h index a9a75a95399..3609398bc3f 100644 --- a/src/Functions/array/FunctionArrayMapped.h +++ b/src/Functions/array/FunctionArrayMapped.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/array/arrayConcat.cpp b/src/Functions/array/arrayConcat.cpp index 1e35932454d..57a783a3244 100644 --- a/src/Functions/array/arrayConcat.cpp +++ b/src/Functions/array/arrayConcat.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp index 5dc8c740c43..916af560c8f 100644 --- a/src/Functions/array/arrayDistinct.cpp +++ b/src/Functions/array/arrayDistinct.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 529d94c04ab..f3d3f558d7b 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayEnumerate.cpp b/src/Functions/array/arrayEnumerate.cpp index c08e0a3bab9..6d59a2c3805 100644 --- a/src/Functions/array/arrayEnumerate.cpp +++ b/src/Functions/array/arrayEnumerate.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayEnumerateExtended.h b/src/Functions/array/arrayEnumerateExtended.h index 78d48b7a476..7e4fe24873a 100644 --- a/src/Functions/array/arrayEnumerateExtended.h +++ b/src/Functions/array/arrayEnumerateExtended.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h index 85b3d751304..2c999415f33 100644 --- a/src/Functions/array/arrayEnumerateRanked.h +++ b/src/Functions/array/arrayEnumerateRanked.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/array/arrayFlatten.cpp b/src/Functions/array/arrayFlatten.cpp index 994f1a2c6cc..3d286aa0bb4 100644 --- a/src/Functions/array/arrayFlatten.cpp +++ b/src/Functions/array/arrayFlatten.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index 55b68155551..f3b279faaef 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index aac918a9181..93c1ee06403 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayJoin.cpp b/src/Functions/array/arrayJoin.cpp index 97cfeb6cd20..da8c4e6e80b 100644 --- a/src/Functions/array/arrayJoin.cpp +++ b/src/Functions/array/arrayJoin.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayPop.h b/src/Functions/array/arrayPop.h index faf5842982e..270b5e20f85 100644 --- a/src/Functions/array/arrayPop.h +++ b/src/Functions/array/arrayPop.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/array/arrayPush.h b/src/Functions/array/arrayPush.h index 6f038eaeed1..3a63e7b631d 100644 --- a/src/Functions/array/arrayPush.h +++ b/src/Functions/array/arrayPush.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index a69ed9dddc8..bbdd142c062 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayReduceInRanges.cpp b/src/Functions/array/arrayReduceInRanges.cpp index 32c02dff14c..bcb34385d17 100644 --- a/src/Functions/array/arrayReduceInRanges.cpp +++ b/src/Functions/array/arrayReduceInRanges.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayResize.cpp b/src/Functions/array/arrayResize.cpp index e771db4bd9a..f8eea06335b 100644 --- a/src/Functions/array/arrayResize.cpp +++ b/src/Functions/array/arrayResize.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayReverse.cpp b/src/Functions/array/arrayReverse.cpp index f7b81f51d7c..fb4a559b37f 100644 --- a/src/Functions/array/arrayReverse.cpp +++ b/src/Functions/array/arrayReverse.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index c77a97a2aa0..567135de884 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 6843bbdd00f..62de746f136 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/arrayWithConstant.cpp b/src/Functions/array/arrayWithConstant.cpp index 5cfb11fddd3..578e8bf2296 100644 --- a/src/Functions/array/arrayWithConstant.cpp +++ b/src/Functions/array/arrayWithConstant.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/emptyArray.cpp b/src/Functions/array/emptyArray.cpp index 4f65380428e..370c51f3e8d 100644 --- a/src/Functions/array/emptyArray.cpp +++ b/src/Functions/array/emptyArray.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/emptyArrayToSingle.cpp b/src/Functions/array/emptyArrayToSingle.cpp index 343d76882e4..be387620e60 100644 --- a/src/Functions/array/emptyArrayToSingle.cpp +++ b/src/Functions/array/emptyArrayToSingle.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/array/hasAllAny.h b/src/Functions/array/hasAllAny.h index 4e03f215b55..04c6558f066 100644 --- a/src/Functions/array/hasAllAny.h +++ b/src/Functions/array/hasAllAny.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 2ec2cc13401..5b9886580dc 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/assumeNotNull.cpp b/src/Functions/assumeNotNull.cpp index 0c1cf515725..019c637e50c 100644 --- a/src/Functions/assumeNotNull.cpp +++ b/src/Functions/assumeNotNull.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/bar.cpp b/src/Functions/bar.cpp index bf0097ba40e..2e26f7af0bb 100644 --- a/src/Functions/bar.cpp +++ b/src/Functions/bar.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/bitmaskToList.cpp b/src/Functions/bitmaskToList.cpp index 1e4d886c127..8c3105724ac 100644 --- a/src/Functions/bitmaskToList.cpp +++ b/src/Functions/bitmaskToList.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/Functions/blockNumber.cpp b/src/Functions/blockNumber.cpp index 5c15d720954..9a57f8a96b0 100644 --- a/src/Functions/blockNumber.cpp +++ b/src/Functions/blockNumber.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/blockSerializedSize.cpp b/src/Functions/blockSerializedSize.cpp index 1cad4cbd773..30f77bbf627 100644 --- a/src/Functions/blockSerializedSize.cpp +++ b/src/Functions/blockSerializedSize.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/blockSize.cpp b/src/Functions/blockSize.cpp index 4476438385b..294686054f0 100644 --- a/src/Functions/blockSize.cpp +++ b/src/Functions/blockSize.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/buildId.cpp b/src/Functions/buildId.cpp index 26b10a8bb6e..cc0c21350ca 100644 --- a/src/Functions/buildId.cpp +++ b/src/Functions/buildId.cpp @@ -1,6 +1,6 @@ #if defined(__ELF__) && !defined(__FreeBSD__) -#include +#include #include #include #include diff --git a/src/Functions/byteSize.cpp b/src/Functions/byteSize.cpp index 463cd4bced0..54a7da59b9c 100644 --- a/src/Functions/byteSize.cpp +++ b/src/Functions/byteSize.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp index 37cc52557d4..791b9d993b4 100644 --- a/src/Functions/coalesce.cpp +++ b/src/Functions/coalesce.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index a024738eab4..c1b9f8e7967 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/connectionId.cpp b/src/Functions/connectionId.cpp index c9e8787dffb..ae04cfd1af6 100644 --- a/src/Functions/connectionId.cpp +++ b/src/Functions/connectionId.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include diff --git a/src/Functions/countDigits.cpp b/src/Functions/countDigits.cpp index b06d5017aab..597a2c625b9 100644 --- a/src/Functions/countDigits.cpp +++ b/src/Functions/countDigits.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/countMatches.h b/src/Functions/countMatches.h index 3bb466f0e31..6ae69520cb9 100644 --- a/src/Functions/countMatches.h +++ b/src/Functions/countMatches.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/currentDatabase.cpp b/src/Functions/currentDatabase.cpp index ed14ccaf447..16eff20cfd5 100644 --- a/src/Functions/currentDatabase.cpp +++ b/src/Functions/currentDatabase.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/currentUser.cpp b/src/Functions/currentUser.cpp index bf63658df4c..22ad49fb29d 100644 --- a/src/Functions/currentUser.cpp +++ b/src/Functions/currentUser.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index 532c4e1c3ec..cbad9dc1ce4 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Functions/defaultValueOfArgumentType.cpp b/src/Functions/defaultValueOfArgumentType.cpp index aa8cfca2083..957d829039d 100644 --- a/src/Functions/defaultValueOfArgumentType.cpp +++ b/src/Functions/defaultValueOfArgumentType.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/defaultValueOfTypeName.cpp b/src/Functions/defaultValueOfTypeName.cpp index ea39f8e1e75..1bf978ab17b 100644 --- a/src/Functions/defaultValueOfTypeName.cpp +++ b/src/Functions/defaultValueOfTypeName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/dumpColumnStructure.cpp b/src/Functions/dumpColumnStructure.cpp index 7bbeee97b85..3189e343beb 100644 --- a/src/Functions/dumpColumnStructure.cpp +++ b/src/Functions/dumpColumnStructure.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/errorCodeToName.cpp b/src/Functions/errorCodeToName.cpp index 5090d41218f..d8e8e0f0d29 100644 --- a/src/Functions/errorCodeToName.cpp +++ b/src/Functions/errorCodeToName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/evalMLMethod.cpp b/src/Functions/evalMLMethod.cpp index 224f85d6815..a69a6f0f550 100644 --- a/src/Functions/evalMLMethod.cpp +++ b/src/Functions/evalMLMethod.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/extractAllGroups.h b/src/Functions/extractAllGroups.h index 0edbeaaf7b1..864a788cf18 100644 --- a/src/Functions/extractAllGroups.h +++ b/src/Functions/extractAllGroups.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/extractTextFromHTML.cpp b/src/Functions/extractTextFromHTML.cpp index 28638696700..6a7bdbeaba9 100644 --- a/src/Functions/extractTextFromHTML.cpp +++ b/src/Functions/extractTextFromHTML.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/filesystem.cpp b/src/Functions/filesystem.cpp index 7f28be10b6e..d264c972656 100644 --- a/src/Functions/filesystem.cpp +++ b/src/Functions/filesystem.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/finalizeAggregation.cpp b/src/Functions/finalizeAggregation.cpp index 48d28f1bd16..b3dfbc0aa15 100644 --- a/src/Functions/finalizeAggregation.cpp +++ b/src/Functions/finalizeAggregation.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index 89b98f1657c..d16985b5b34 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/formatReadable.h b/src/Functions/formatReadable.h index e4681163e8a..6cf4fadbf05 100644 --- a/src/Functions/formatReadable.h +++ b/src/Functions/formatReadable.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/formatReadableTimeDelta.cpp b/src/Functions/formatReadableTimeDelta.cpp index 83af6968bac..69dbaa71041 100644 --- a/src/Functions/formatReadableTimeDelta.cpp +++ b/src/Functions/formatReadableTimeDelta.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/Functions/formatRow.cpp b/src/Functions/formatRow.cpp index 9012137b334..2d7739a02d6 100644 --- a/src/Functions/formatRow.cpp +++ b/src/Functions/formatRow.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp index 4de34c1f43f..4a3b04aaa05 100644 --- a/src/Functions/formatString.cpp +++ b/src/Functions/formatString.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/fuzzBits.cpp b/src/Functions/fuzzBits.cpp index b6732d3211c..0884f586082 100644 --- a/src/Functions/fuzzBits.cpp +++ b/src/Functions/fuzzBits.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 23df9d246ec..6030cbeea65 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/geohashesInBox.cpp b/src/Functions/geohashesInBox.cpp index 55d89649dc8..c0629ab9655 100644 --- a/src/Functions/geohashesInBox.cpp +++ b/src/Functions/geohashesInBox.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/getMacro.cpp b/src/Functions/getMacro.cpp index 10d89a2e7d1..c869685af42 100644 --- a/src/Functions/getMacro.cpp +++ b/src/Functions/getMacro.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/getScalar.cpp b/src/Functions/getScalar.cpp index 8d6aa70bdb3..a29abd257e7 100644 --- a/src/Functions/getScalar.cpp +++ b/src/Functions/getScalar.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/getSetting.cpp b/src/Functions/getSetting.cpp index b90fa48dc73..0206de33125 100644 --- a/src/Functions/getSetting.cpp +++ b/src/Functions/getSetting.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/getSizeOfEnumType.cpp b/src/Functions/getSizeOfEnumType.cpp index e58faacb187..7db11d41d0f 100644 --- a/src/Functions/getSizeOfEnumType.cpp +++ b/src/Functions/getSizeOfEnumType.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/globalVariable.cpp b/src/Functions/globalVariable.cpp index 6a5fda3b831..c141ceb2692 100644 --- a/src/Functions/globalVariable.cpp +++ b/src/Functions/globalVariable.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index d9ead27ac9f..756b76859ae 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index 05967133406..da7e59d69e3 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/hostName.cpp b/src/Functions/hostName.cpp index f09069a7399..0aba155bb36 100644 --- a/src/Functions/hostName.cpp +++ b/src/Functions/hostName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/identity.cpp b/src/Functions/identity.cpp index bdc74fbaf19..d15d9e1f710 100644 --- a/src/Functions/identity.cpp +++ b/src/Functions/identity.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index e57b621acad..ec3447ffb81 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/ifNotFinite.cpp b/src/Functions/ifNotFinite.cpp index d85fc4ee4ca..a5e3131117e 100644 --- a/src/Functions/ifNotFinite.cpp +++ b/src/Functions/ifNotFinite.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp index eba3f74753c..82b63f4dbda 100644 --- a/src/Functions/ifNull.cpp +++ b/src/Functions/ifNull.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/ignore.cpp b/src/Functions/ignore.cpp index cdb267d7a2a..45079639835 100644 --- a/src/Functions/ignore.cpp +++ b/src/Functions/ignore.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index d891b84fe22..827e0212396 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/indexHint.cpp b/src/Functions/indexHint.cpp index 2f1fc40bee9..d2a9a0abb8c 100644 --- a/src/Functions/indexHint.cpp +++ b/src/Functions/indexHint.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/initializeAggregation.cpp b/src/Functions/initializeAggregation.cpp index 0f19dcdc3c6..aea97a48037 100644 --- a/src/Functions/initializeAggregation.cpp +++ b/src/Functions/initializeAggregation.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/isConstant.cpp b/src/Functions/isConstant.cpp index b2440aec13f..3069ec79ae1 100644 --- a/src/Functions/isConstant.cpp +++ b/src/Functions/isConstant.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index ae8cff19192..d409afaf234 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/isIPAddressContainedIn.cpp b/src/Functions/isIPAddressContainedIn.cpp index 06f439f956c..0886ef55e7b 100644 --- a/src/Functions/isIPAddressContainedIn.cpp +++ b/src/Functions/isIPAddressContainedIn.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/isNotNull.cpp b/src/Functions/isNotNull.cpp index 05964716053..81c870a6303 100644 --- a/src/Functions/isNotNull.cpp +++ b/src/Functions/isNotNull.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp index 074f52c3cb6..de840dab2bf 100644 --- a/src/Functions/isNull.cpp +++ b/src/Functions/isNull.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/isZeroOrNull.cpp b/src/Functions/isZeroOrNull.cpp index 9152c2692d4..a2a06af7569 100644 --- a/src/Functions/isZeroOrNull.cpp +++ b/src/Functions/isZeroOrNull.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/logTrace.cpp b/src/Functions/logTrace.cpp index 17ddcba0929..2ccc4d2ffd2 100644 --- a/src/Functions/logTrace.cpp +++ b/src/Functions/logTrace.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/lowCardinalityIndices.cpp b/src/Functions/lowCardinalityIndices.cpp index 4e560eaf0fa..580e7381955 100644 --- a/src/Functions/lowCardinalityIndices.cpp +++ b/src/Functions/lowCardinalityIndices.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/lowCardinalityKeys.cpp b/src/Functions/lowCardinalityKeys.cpp index ae916cecebe..46128267871 100644 --- a/src/Functions/lowCardinalityKeys.cpp +++ b/src/Functions/lowCardinalityKeys.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 7e1ea33a21c..f1b08dc75b7 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index bc4fc7a9aad..4676b263f39 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include namespace DB diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index d49d4ab70dc..785c5817176 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/nullIf.cpp b/src/Functions/nullIf.cpp index 66c35a26978..14f2f72fe61 100644 --- a/src/Functions/nullIf.cpp +++ b/src/Functions/nullIf.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp index 4238f1f0bf1..cf679452da1 100644 --- a/src/Functions/partitionId.cpp +++ b/src/Functions/partitionId.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/pointInEllipses.cpp b/src/Functions/pointInEllipses.cpp index b10bbd40197..f7bfc24559c 100644 --- a/src/Functions/pointInEllipses.cpp +++ b/src/Functions/pointInEllipses.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/randomFixedString.cpp b/src/Functions/randomFixedString.cpp index d2484bfc2b4..13996eee677 100644 --- a/src/Functions/randomFixedString.cpp +++ b/src/Functions/randomFixedString.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/randomPrintableASCII.cpp b/src/Functions/randomPrintableASCII.cpp index e6491945dfd..ba8c16aa689 100644 --- a/src/Functions/randomPrintableASCII.cpp +++ b/src/Functions/randomPrintableASCII.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/randomString.cpp b/src/Functions/randomString.cpp index 758abb15ec9..10795626326 100644 --- a/src/Functions/randomString.cpp +++ b/src/Functions/randomString.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index 60476a89c26..018c591a1fe 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index 430fcc81181..423ed53c53f 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/replicate.cpp b/src/Functions/replicate.cpp index f023f813828..796d5645a2e 100644 --- a/src/Functions/replicate.cpp +++ b/src/Functions/replicate.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/Functions/replicate.h b/src/Functions/replicate.h index cd401f1d267..71b39e9e60e 100644 --- a/src/Functions/replicate.h +++ b/src/Functions/replicate.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include namespace DB diff --git a/src/Functions/rowNumberInAllBlocks.cpp b/src/Functions/rowNumberInAllBlocks.cpp index ca3df2f0c40..9c358aec8f0 100644 --- a/src/Functions/rowNumberInAllBlocks.cpp +++ b/src/Functions/rowNumberInAllBlocks.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/rowNumberInBlock.cpp b/src/Functions/rowNumberInBlock.cpp index 18d1b65588b..48fa472e1dd 100644 --- a/src/Functions/rowNumberInBlock.cpp +++ b/src/Functions/rowNumberInBlock.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index e7f7d51a3b8..1eab573b046 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/runningConcurrency.cpp b/src/Functions/runningConcurrency.cpp index 452e6542d83..022e2be5f6c 100644 --- a/src/Functions/runningConcurrency.cpp +++ b/src/Functions/runningConcurrency.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/runningDifference.h b/src/Functions/runningDifference.h index 34f4c4033e1..f87c57af043 100644 --- a/src/Functions/runningDifference.h +++ b/src/Functions/runningDifference.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/sleep.h b/src/Functions/sleep.h index 3ffb0f43925..c0aad0b3820 100644 --- a/src/Functions/sleep.h +++ b/src/Functions/sleep.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include #include #include diff --git a/src/Functions/substring.cpp b/src/Functions/substring.cpp index 5e5e3b64280..5ce75035475 100644 --- a/src/Functions/substring.cpp +++ b/src/Functions/substring.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/subtractDays.cpp b/src/Functions/subtractDays.cpp index 7e4dbb2ecc1..21966a15311 100644 --- a/src/Functions/subtractDays.cpp +++ b/src/Functions/subtractDays.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractHours.cpp b/src/Functions/subtractHours.cpp index 1a261ad3d36..e71c9d74a01 100644 --- a/src/Functions/subtractHours.cpp +++ b/src/Functions/subtractHours.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractMinutes.cpp b/src/Functions/subtractMinutes.cpp index 9ea92a49cf0..ba9d593a64d 100644 --- a/src/Functions/subtractMinutes.cpp +++ b/src/Functions/subtractMinutes.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractMonths.cpp b/src/Functions/subtractMonths.cpp index effdb077bcf..64eeba2ce86 100644 --- a/src/Functions/subtractMonths.cpp +++ b/src/Functions/subtractMonths.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractQuarters.cpp b/src/Functions/subtractQuarters.cpp index 3b6ff64b719..6c066ed17a1 100644 --- a/src/Functions/subtractQuarters.cpp +++ b/src/Functions/subtractQuarters.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractSeconds.cpp b/src/Functions/subtractSeconds.cpp index d63a1f41d9c..81a7f7e2df1 100644 --- a/src/Functions/subtractSeconds.cpp +++ b/src/Functions/subtractSeconds.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractWeeks.cpp b/src/Functions/subtractWeeks.cpp index 74e422acbbf..55b52043dd0 100644 --- a/src/Functions/subtractWeeks.cpp +++ b/src/Functions/subtractWeeks.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/subtractYears.cpp b/src/Functions/subtractYears.cpp index 93fcb44baf0..241142722d5 100644 --- a/src/Functions/subtractYears.cpp +++ b/src/Functions/subtractYears.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index 1e22295edb7..1b56cc0d188 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/tid.cpp b/src/Functions/tid.cpp index 56aad782cdd..404eff862b3 100644 --- a/src/Functions/tid.cpp +++ b/src/Functions/tid.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/timeSlot.cpp b/src/Functions/timeSlot.cpp index 7819a7c1341..afe99c86eb2 100644 --- a/src/Functions/timeSlot.cpp +++ b/src/Functions/timeSlot.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 8530c41bf2f..b64d2687b05 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Functions/timezone.cpp b/src/Functions/timezone.cpp index 27ed19a22aa..67f7462fc95 100644 --- a/src/Functions/timezone.cpp +++ b/src/Functions/timezone.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/timezoneOf.cpp b/src/Functions/timezoneOf.cpp index 9606951d006..0e6fa1bb213 100644 --- a/src/Functions/timezoneOf.cpp +++ b/src/Functions/timezoneOf.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/toColumnTypeName.cpp b/src/Functions/toColumnTypeName.cpp index 44d27f9fd48..0b73fe772d1 100644 --- a/src/Functions/toColumnTypeName.cpp +++ b/src/Functions/toColumnTypeName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/toCustomWeek.cpp b/src/Functions/toCustomWeek.cpp index 715e899b5fe..5ba0b8e8b2a 100644 --- a/src/Functions/toCustomWeek.cpp +++ b/src/Functions/toCustomWeek.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Functions/toFixedString.h b/src/Functions/toFixedString.h index 1d6b79cb583..fe437a24303 100644 --- a/src/Functions/toFixedString.h +++ b/src/Functions/toFixedString.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/src/Functions/toLowCardinality.cpp b/src/Functions/toLowCardinality.cpp index 53cf1e9fea8..983e66d1007 100644 --- a/src/Functions/toLowCardinality.cpp +++ b/src/Functions/toLowCardinality.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/toModifiedJulianDay.cpp b/src/Functions/toModifiedJulianDay.cpp index fe03fcb1bda..a44979b52ff 100644 --- a/src/Functions/toModifiedJulianDay.cpp +++ b/src/Functions/toModifiedJulianDay.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/toNullable.cpp b/src/Functions/toNullable.cpp index 0f9a4a437f6..5e383893476 100644 --- a/src/Functions/toNullable.cpp +++ b/src/Functions/toNullable.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index b7716501d02..ee592fbb1e3 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index 58ebb67ce17..551e07a8354 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include #include diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index 556cb7d7877..4c7b00e74ed 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 8e481700a9e..43be4c4582a 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -4,7 +4,7 @@ #include -#include +#include #include diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index 1e4d12b82af..1debc2cb6a0 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index e527fcd7a73..dda034ee911 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/tupleElement.cpp b/src/Functions/tupleElement.cpp index 8dadda3f14f..97e2f70fccf 100644 --- a/src/Functions/tupleElement.cpp +++ b/src/Functions/tupleElement.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/uptime.cpp b/src/Functions/uptime.cpp index a5561c709e8..02454df4de5 100644 --- a/src/Functions/uptime.cpp +++ b/src/Functions/uptime.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/version.cpp b/src/Functions/version.cpp index 833816d6057..4e0ddf60975 100644 --- a/src/Functions/version.cpp +++ b/src/Functions/version.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/visibleWidth.cpp b/src/Functions/visibleWidth.cpp index 137b39994b1..6e96a4844ce 100644 --- a/src/Functions/visibleWidth.cpp +++ b/src/Functions/visibleWidth.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/yesterday.cpp b/src/Functions/yesterday.cpp index 4fdbf34a968..737552e192e 100644 --- a/src/Functions/yesterday.cpp +++ b/src/Functions/yesterday.cpp @@ -4,7 +4,7 @@ #include -#include +#include #include diff --git a/src/Interpreters/JIT/CompileDAG.h b/src/Interpreters/JIT/CompileDAG.h index 87228c83d5c..ee9cc5cc83a 100644 --- a/src/Interpreters/JIT/CompileDAG.h +++ b/src/Interpreters/JIT/CompileDAG.h @@ -12,7 +12,7 @@ #include #include -#include +#include namespace llvm { From 84fc1e0c3d2e8ea654e1879898601e232d13cf69 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 17 May 2021 10:32:29 +0300 Subject: [PATCH 078/202] Fixed style check --- src/Functions/ya.make | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index fcc92da6fcc..2a541369ff4 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -74,7 +74,6 @@ SRCS( GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp GeoHash.cpp IFunction.cpp - IFunctionOld.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp From 448fe5a6f973b25e575489dc64d21f02795666a2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 17 May 2021 11:04:29 +0300 Subject: [PATCH 079/202] Updated comment message --- tests/performance/column_column_comparison.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/performance/column_column_comparison.xml b/tests/performance/column_column_comparison.xml index 15354c86f55..bacab767f49 100644 --- a/tests/performance/column_column_comparison.xml +++ b/tests/performance/column_column_comparison.xml @@ -1,5 +1,10 @@ +[原始文章] (https://clickhouse.tech/docs/en/operations/backup/) From a2fa599e036ca5474ace4625f70e738c1bc103f5 Mon Sep 17 00:00:00 2001 From: huzhichengdd <83016994+huzhichengdd@users.noreply.github.com> Date: Wed, 19 May 2021 19:15:21 +0800 Subject: [PATCH 143/202] Update backup.md --- docs/zh/operations/backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/backup.md b/docs/zh/operations/backup.md index 317a5bebec7..6d517e6ccb3 100644 --- a/docs/zh/operations/backup.md +++ b/docs/zh/operations/backup.md @@ -32,7 +32,7 @@ toc_title: "\u6570\u636E\u5907\u4EFD" ## part操作 {#manipulations-with-parts} -ClickHouse允许使用 `ALTER TABLE ... FREEZE PARTITION ...` 查询以创建表分区的本地副本。 这是利用硬链接(hardlink)到 `/var/lib/clickhouse/shadow/` 文件夹中实现的，所以它通常不会因为旧数据而占用额外的磁盘空间。 创建的文件副本不由ClickHouse服务器处理，所以你可以把它们留在那里：你将有一个简单的备份，不需要任何额外的外部系统，但它仍然容易出现硬件问题。 出于这个原因，最好将它们远程复制到另一个位置，然后删除本地副本。 分布式文件系统和对象存储仍然是一个不错的选择，但是具有足够大容量的正常附加文件服务器也可以工作（在这种情况下，传输将通过网络文件系统或者也许是 [rsync] (https://en.wikipedia.org/wiki/Rsync)来进行). +ClickHouse允许使用 `ALTER TABLE ... FREEZE PARTITION ...` 查询以创建表分区的本地副本。 这是利用硬链接(hardlink)到 `/var/lib/clickhouse/shadow/` 文件夹中实现的，所以它通常不会因为旧数据而占用额外的磁盘空间。 创建的文件副本不由ClickHouse服务器处理，所以你可以把它们留在那里：你将有一个简单的备份，不需要任何额外的外部系统，但它仍然容易出现硬件问题。 出于这个原因，最好将它们远程复制到另一个位置，然后删除本地副本。 分布式文件系统和对象存储仍然是一个不错的选择，但是具有足够大容量的正常附加文件服务器也可以工作（在这种情况下，传输将通过网络文件系统或者也许是 [rsync] (https://en.wikipedia.org/wiki/Rsync) 来进行). 数据可以使用 `ALTER TABLE ... ATTACH PARTITION ...` 从备份中恢复。 From aa58fc096ce214385a431417ee9f85f1a8d7b3d2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 19 May 2021 14:44:39 +0300 Subject: [PATCH 144/202] done --- src/Common/ya.make | 1 + src/Common/ya.make.in | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Common/ya.make b/src/Common/ya.make index f12b17827f7..dde1e6ae013 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -18,6 +18,7 @@ PEERDIR( contrib/libs/openssl contrib/libs/poco/NetSSL_OpenSSL contrib/libs/re2 + contrib/libs/cxxsupp/libcxxabi-parts contrib/restricted/dragonbox ) diff --git a/src/Common/ya.make.in b/src/Common/ya.make.in index fd6a805891e..459266a54e7 100644 --- a/src/Common/ya.make.in +++ b/src/Common/ya.make.in @@ -17,6 +17,7 @@ PEERDIR( contrib/libs/openssl contrib/libs/poco/NetSSL_OpenSSL contrib/libs/re2 + contrib/libs/cxxsupp/libcxxabi-parts contrib/restricted/dragonbox ) From c07043fd971b662406153730bb2f2c4afae3fe51 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 19 May 2021 15:16:18 +0300 Subject: [PATCH 145/202] Try fix more cases. --- .../ClusterProxy/SelectStreamFactory.cpp | 20 ++++++++++++------- ..._Distributed__shard_num_GROUP_BY.reference | 2 ++ .../01860_Distributed__shard_num_GROUP_BY.sql | 4 ++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 3d7aa589225..c98535c8bb0 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -92,6 +92,11 @@ Block evaluateConstantGroupByKeysWithShardNumber( { Block res; + ColumnWithTypeAndName shard_num_col; + shard_num_col.type = std::make_shared(); + shard_num_col.column = shard_num_col.type->createColumnConst(0, shard_num); + shard_num_col.name = "_shard_num"; + if (auto group_by = query_ast->as().groupBy()) { for (const auto & elem : group_by->children) @@ -108,19 +113,20 @@ Block evaluateConstantGroupByKeysWithShardNumber( if (required_columns.size() != 1 || required_columns.count("_shard_num") == 0) continue; - auto type = std::make_shared(); - auto col = type->createColumnConst(1, shard_num); - std::string name = "_shard_num"; - - Block block({ColumnWithTypeAndName{col, type, name}}); - auto syntax_result = TreeRewriter(context).analyze(ast, {NameAndTypePair{name, type}}); - ExpressionAnalyzer(ast, syntax_result, context).getActions(true)->execute(block); + Block block({shard_num_col}); + auto syntax_result = TreeRewriter(context).analyze(ast, {NameAndTypePair{shard_num_col.name, shard_num_col.type}}); + ExpressionAnalyzer(ast, syntax_result, context).getActions(true, false)->execute(block); res.insert(block.getByName(key_name)); } } } + /// We always add _shard_num constant just in case. + /// For initial query it is considered as a column from table, and may be required by intermediate block. + if (!res.has(shard_num_col.name)) + res.insert(std::move(shard_num_col)); + return res; } diff --git a/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.reference b/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.reference index decc92f3ac0..fa0301316ae 100644 --- a/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.reference +++ b/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.reference @@ -14,3 +14,5 @@ 2 1 2 +1 1 +2 1 diff --git a/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.sql b/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.sql index 05b82ce4064..91215fd8ee6 100644 --- a/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.sql +++ b/tests/queries/0_stateless/01860_Distributed__shard_num_GROUP_BY.sql @@ -10,3 +10,7 @@ SELECT _shard_num + dummy s, count() FROM remote('127.0.0.{1,2}', system.one) GR SELECT _shard_num FROM remote('127.0.0.{1,2}', system.one) ORDER BY _shard_num; SELECT _shard_num s FROM remote('127.0.0.{1,2}', system.one) ORDER BY _shard_num; + +SELECT _shard_num s, count() FROM remote('127.0.0.{1,2}', system.one) GROUP BY s order by s; + +select materialize(_shard_num), * from remote('127.{1,2}', system.one) limit 1 by dummy format Null; From 9f28d0966a3b4f9ec2783a78cefee28996a4c154 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 19 May 2021 15:43:10 +0300 Subject: [PATCH 146/202] Fix typos. --- src/Interpreters/ClusterProxy/SelectStreamFactory.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index c98535c8bb0..0c9d42e1381 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -77,10 +77,10 @@ namespace /// Special support for the case when `_shard_num` column is used in GROUP BY key expression. /// This column is a constant for shard. -/// Constant expression with this colum may be removed from intermediate header. +/// Constant expression with this column may be removed from intermediate header. /// However, this column is not constant for initiator, and it expect intermediate header has it. /// -/// To fix it, the follwing trick is applied. +/// To fix it, the following trick is applied. /// We check all GROUP BY keys which depend only on `_shard_num`. /// Calculate such expression for current shard if it is used in header. /// Those columns will be added to modified header as already known constants. From 5a359a15bcba8b924422504b4ced48806035e4f3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 19 May 2021 15:57:37 +0300 Subject: [PATCH 147/202] Fixed Arcadia after IFunctionOverloadResolver interface refactoring --- src/Functions/IFunction.cpp | 8 +++++--- src/Functions/IFunction.h | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp index 262f5f7f0a8..7f019ccc8f7 100644 --- a/src/Functions/IFunction.cpp +++ b/src/Functions/IFunction.cpp @@ -266,9 +266,11 @@ void IFunctionOverloadResolver::checkNumberOfArguments(size_t number_of_argument size_t expected_number_of_arguments = getNumberOfArguments(); if (number_of_arguments != expected_number_of_arguments) - throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " - + toString(number_of_arguments) + ", should be " + toString(expected_number_of_arguments), - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be {}", + getName(), + toString(number_of_arguments), + toString(expected_number_of_arguments)); } DataTypePtr IFunctionOverloadResolver::getReturnType(const ColumnsWithTypeAndName & arguments) const diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index fe3ec21afa1..7542451a81a 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -245,6 +245,8 @@ public: void getLambdaArgumentTypes(DataTypes & arguments) const; + void checkNumberOfArguments(size_t number_of_arguments) const; + /// Get the main function name. virtual String getName() const = 0; @@ -319,8 +321,6 @@ protected: private: - void checkNumberOfArguments(size_t number_of_arguments) const; - DataTypePtr getReturnTypeWithoutLowCardinality(const ColumnsWithTypeAndName & arguments) const; }; From 96992990834a8cf2af7970f70b71202cdb895921 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 19 May 2021 16:11:38 +0300 Subject: [PATCH 148/202] Fix build --- src/Interpreters/ExpressionActions.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 4f4281bd150..603aeba8699 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -45,7 +45,10 @@ namespace ErrorCodes ExpressionActions::~ExpressionActions() = default; -ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_, CompileExpressions compile_expressions) +ExpressionActions::ExpressionActions( + ActionsDAGPtr actions_dag_, + const ExpressionActionsSettings & settings_, + CompileExpressions compile_expressions [[maybe_unused]]) : settings(settings_) { actions_dag = actions_dag_->clone(); From 0ca9ba19c06134e0e4b60d8f4c5c64911b502508 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 19 May 2021 16:14:28 +0300 Subject: [PATCH 149/202] Remove performance test --- .../performance/column_column_comparison.xml | 38 ------------------- .../00911_tautological_compare.sql | 33 ++++++++++++++++ 2 files changed, 33 insertions(+), 38 deletions(-) delete mode 100644 tests/performance/column_column_comparison.xml diff --git a/tests/performance/column_column_comparison.xml b/tests/performance/column_column_comparison.xml deleted file mode 100644 index bacab767f49..00000000000 --- a/tests/performance/column_column_comparison.xml +++ /dev/null @@ -1,38 +0,0 @@ - diff --git a/tests/queries/0_stateless/00911_tautological_compare.sql b/tests/queries/0_stateless/00911_tautological_compare.sql index a10aff449bb..bcbbbeb514b 100644 --- a/tests/queries/0_stateless/00911_tautological_compare.sql +++ b/tests/queries/0_stateless/00911_tautological_compare.sql @@ -14,3 +14,36 @@ -- SELECT count() FROM system.numbers WHERE SHA256(toString(number)) != SHA256(toString(number)); -- SELECT count() FROM system.numbers WHERE SHA256(toString(number)) != SHA256(toString(number)) AND rand() > 10; + +-- column_column_comparison.xml +-- +-- +-- comparison +-- + +-- +-- hits_100m_single +-- + + +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- +-- + +-- From 09cb46781258824ca1553371197ee8cbe79751dd Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 19 May 2021 16:14:33 +0300 Subject: [PATCH 150/202] Update StorageDistributed.cpp --- src/Storages/StorageDistributed.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 718a15888ea..3ec8f472ed3 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -281,7 +281,9 @@ void replaceConstantExpressions( visitor.visit(node); } -/// Returns one of the following: +/// This is the implementation of optimize_distributed_group_by_sharding_key. +/// It returns up to which stage the query can be processed on a shard, which +/// is one of the following: /// - QueryProcessingStage::Complete /// - QueryProcessingStage::WithMergeableStateAfterAggregation /// - none (in this case regular WithMergeableState should be used) From 33f816d28d0b26c2f5bbbf88f0b0afc11846fa21 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Wed, 19 May 2021 16:20:23 +0300 Subject: [PATCH 151/202] Allow empty HTTP headers --- src/Common/StringUtils/StringUtils.h | 2 +- src/Server/HTTP/ReadHeaders.cpp | 3 --- tests/queries/0_stateless/01399_http_request_headers.sh | 1 + 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Common/StringUtils/StringUtils.h b/src/Common/StringUtils/StringUtils.h index cb2227f01a8..20c0a5ca380 100644 --- a/src/Common/StringUtils/StringUtils.h +++ b/src/Common/StringUtils/StringUtils.h @@ -123,7 +123,7 @@ inline bool isWhitespaceASCII(char c) /// Since |isWhiteSpaceASCII()| is used inside algorithms it's easier to implement another function than add extra argument. inline bool isWhitespaceASCIIOneLine(char c) { - return c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'; + return c == ' ' || c == '\t' || c == '\f' || c == '\v'; } inline bool isControlASCII(char c) diff --git a/src/Server/HTTP/ReadHeaders.cpp b/src/Server/HTTP/ReadHeaders.cpp index 2fc2de8321a..b7057501064 100644 --- a/src/Server/HTTP/ReadHeaders.cpp +++ b/src/Server/HTTP/ReadHeaders.cpp @@ -68,9 +68,6 @@ void readHeaders( if (in.eof()) throw Poco::Net::MessageException("Field is invalid"); - if (value.empty()) - throw Poco::Net::MessageException("Field value is empty"); - if (ch == '\n') throw Poco::Net::MessageException("No CRLF found"); diff --git a/tests/queries/0_stateless/01399_http_request_headers.sh b/tests/queries/0_stateless/01399_http_request_headers.sh index 9b07f018230..f06e7ffc32b 100755 --- a/tests/queries/0_stateless/01399_http_request_headers.sh +++ b/tests/queries/0_stateless/01399_http_request_headers.sh @@ -4,6 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'EmptyHeader;' -d 'SELECT 1' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-User: default' -d 'SELECT 1' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-User: header_test' -d 'SELECT 1' | grep -o 'Code: 516' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -H 'X-ClickHouse-Key: ' -d 'SELECT 1' From d92b1f0e9ad4d2bf20ab1a6a1861fe30f0bcc21c Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 19 May 2021 08:56:08 +0000 Subject: [PATCH 152/202] Fix mutation of StorageMemory fix fix --- src/Storages/StorageMemory.cpp | 9 +++++- ...1867_fix_storage_memory_mutation.reference | 0 .../01867_fix_storage_memory_mutation.sql | 32 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01867_fix_storage_memory_mutation.reference create mode 100644 tests/queries/0_stateless/01867_fix_storage_memory_mutation.sql diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 289a17366bb..1deb6a6618f 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -262,7 +262,14 @@ void StorageMemory::mutate(const MutationCommands & commands, ContextPtr context auto metadata_snapshot = getInMemoryMetadataPtr(); auto storage = getStorageID(); auto storage_ptr = DatabaseCatalog::instance().getTable(storage, context); - auto interpreter = std::make_unique(storage_ptr, metadata_snapshot, commands, context, true); + + /// When max_threads > 1, the order of returning blocks is uncentain, + /// which will lead to inconsistency after updateBlockData. + auto new_context = Context::createCopy(context); + new_context->setSetting("max_streams_to_max_threads_ratio", 1); + new_context->setSetting("max_threads", 1); + + auto interpreter = std::make_unique(storage_ptr, metadata_snapshot, commands, new_context, true); auto in = interpreter->execute(); in->readPrefix(); diff --git a/tests/queries/0_stateless/01867_fix_storage_memory_mutation.reference b/tests/queries/0_stateless/01867_fix_storage_memory_mutation.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01867_fix_storage_memory_mutation.sql b/tests/queries/0_stateless/01867_fix_storage_memory_mutation.sql new file mode 100644 index 00000000000..4cb80036d73 --- /dev/null +++ b/tests/queries/0_stateless/01867_fix_storage_memory_mutation.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS mem_test; + +CREATE TABLE mem_test +( + `a` Int64, + `b` Int64 +) +ENGINE = Memory; + +SET max_block_size = 3; + +INSERT INTO mem_test SELECT + number, + number +FROM numbers(100); + +ALTER TABLE mem_test + UPDATE a = 0 WHERE b = 99; +ALTER TABLE mem_test + UPDATE a = 0 WHERE b = 99; +ALTER TABLE mem_test + UPDATE a = 0 WHERE b = 99; +ALTER TABLE mem_test + UPDATE a = 0 WHERE b = 99; +ALTER TABLE mem_test + UPDATE a = 0 WHERE b = 99; + +SELECT * +FROM mem_test +FORMAT Null; + +DROP TABLE mem_test; From c50909468f538fd3507b8333d6d68ace903eff2a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 19 May 2021 17:32:07 +0300 Subject: [PATCH 153/202] Updated ExpressionActionSettings --- src/Core/Settings.h | 2 +- src/DataStreams/AddingDefaultBlockOutputStream.cpp | 2 +- src/DataStreams/AddingDefaultsBlockInputStream.cpp | 2 +- src/Interpreters/ActionsVisitor.cpp | 2 +- src/Interpreters/ExpressionActions.cpp | 7 ++----- src/Interpreters/ExpressionActions.h | 8 +------- src/Interpreters/ExpressionActionsSettings.cpp | 9 +++++---- src/Interpreters/ExpressionActionsSettings.h | 14 +++++++++++--- src/Interpreters/ExpressionAnalyzer.cpp | 6 +++--- src/Interpreters/InterpreterInsertQuery.cpp | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 10 +++++----- src/Processors/QueryPlan/ArrayJoinStep.cpp | 3 ++- .../QueryPlan/BuildQueryPipelineSettings.cpp | 2 +- src/Processors/QueryPlan/ExpressionStep.cpp | 4 ++-- src/Processors/QueryPlan/FilterStep.cpp | 4 ++-- src/Processors/QueryPlan/TotalsHavingStep.cpp | 2 +- src/Storages/StorageMerge.cpp | 5 ++--- src/Storages/StorageTableFunction.h | 3 +-- 18 files changed, 43 insertions(+), 44 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ea2019a4ff1..2d0e2eb2f9c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -102,7 +102,7 @@ class IColumn; \ M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \ M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \ - M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ + M(UInt64, min_count_to_compile_expression, 0, "The number of identical expressions before they are JIT-compiled", 0) \ M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ diff --git a/src/DataStreams/AddingDefaultBlockOutputStream.cpp b/src/DataStreams/AddingDefaultBlockOutputStream.cpp index e60f7ffc069..6f7975d492d 100644 --- a/src/DataStreams/AddingDefaultBlockOutputStream.cpp +++ b/src/DataStreams/AddingDefaultBlockOutputStream.cpp @@ -15,7 +15,7 @@ AddingDefaultBlockOutputStream::AddingDefaultBlockOutputStream( : output(output_), header(header_) { auto dag = addMissingDefaults(header_, output->getHeader().getNamesAndTypesList(), columns_, context_, null_as_default_); - adding_defaults_actions = std::make_shared(std::move(dag), ExpressionActionsSettings::fromContext(context_), CompileExpressions::yes); + adding_defaults_actions = std::make_shared(std::move(dag), ExpressionActionsSettings::fromContext(context_, CompileExpressions::yes)); } void AddingDefaultBlockOutputStream::write(const Block & block) diff --git a/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 9853f23294f..81be24439a5 100644 --- a/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -174,7 +174,7 @@ Block AddingDefaultsBlockInputStream::readImpl() auto dag = evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), columns, context, false); if (dag) { - auto actions = std::make_shared(std::move(dag), ExpressionActionsSettings::fromContext(context), CompileExpressions::yes); + auto actions = std::make_shared(std::move(dag), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)); actions->execute(evaluate_block); } diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 341e3f8877a..44eb7c902cd 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -1015,7 +1015,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & auto lambda_actions = std::make_shared( lambda_dag, - ExpressionActionsSettings::fromContext(data.getContext()), CompileExpressions::yes); + ExpressionActionsSettings::fromContext(data.getContext(), CompileExpressions::yes)); DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 603aeba8699..bd06c753319 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -45,16 +45,13 @@ namespace ErrorCodes ExpressionActions::~ExpressionActions() = default; -ExpressionActions::ExpressionActions( - ActionsDAGPtr actions_dag_, - const ExpressionActionsSettings & settings_, - CompileExpressions compile_expressions [[maybe_unused]]) +ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_) : settings(settings_) { actions_dag = actions_dag_->clone(); #if USE_EMBEDDED_COMPILER - if (settings.compile_expressions && compile_expressions == CompileExpressions::yes) + if (settings.can_compile_expressions && settings.compile_expressions == CompileExpressions::yes) actions_dag->compileExpressions(settings.min_count_to_compile_expression); #endif diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 04be573b50e..c446b339072 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -30,12 +30,6 @@ using ArrayJoinActionPtr = std::shared_ptr; class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; -enum class CompileExpressions: uint8_t -{ - no = 0, - yes = 1, -}; - /// Sequence of actions on the block. /// Is used to calculate expressions. /// @@ -89,7 +83,7 @@ private: public: ExpressionActions() = delete; ~ExpressionActions(); - explicit ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_ = {}, CompileExpressions compile_expressions = CompileExpressions::no); + explicit ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_ = {}); ExpressionActions(const ExpressionActions &) = default; ExpressionActions & operator=(const ExpressionActions &) = default; diff --git a/src/Interpreters/ExpressionActionsSettings.cpp b/src/Interpreters/ExpressionActionsSettings.cpp index a9495628d6f..550aa4d339c 100644 --- a/src/Interpreters/ExpressionActionsSettings.cpp +++ b/src/Interpreters/ExpressionActionsSettings.cpp @@ -6,20 +6,21 @@ namespace DB { -ExpressionActionsSettings ExpressionActionsSettings::fromSettings(const Settings & from) +ExpressionActionsSettings ExpressionActionsSettings::fromSettings(const Settings & from, CompileExpressions compile_expressions) { ExpressionActionsSettings settings; - settings.compile_expressions = from.compile_expressions; + settings.can_compile_expressions = from.compile_expressions; settings.min_count_to_compile_expression = from.min_count_to_compile_expression; settings.max_temporary_columns = from.max_temporary_columns; settings.max_temporary_non_const_columns = from.max_temporary_non_const_columns; + settings.compile_expressions = compile_expressions; return settings; } -ExpressionActionsSettings ExpressionActionsSettings::fromContext(ContextPtr from) +ExpressionActionsSettings ExpressionActionsSettings::fromContext(ContextPtr from, CompileExpressions compile_expressions) { - return fromSettings(from->getSettingsRef()); + return fromSettings(from->getSettingsRef(), compile_expressions); } } diff --git a/src/Interpreters/ExpressionActionsSettings.h b/src/Interpreters/ExpressionActionsSettings.h index 06351136a9a..26532128805 100644 --- a/src/Interpreters/ExpressionActionsSettings.h +++ b/src/Interpreters/ExpressionActionsSettings.h @@ -9,16 +9,24 @@ namespace DB struct Settings; +enum class CompileExpressions: uint8_t +{ + no = 0, + yes = 1, +}; + struct ExpressionActionsSettings { - bool compile_expressions = false; + bool can_compile_expressions = false; size_t min_count_to_compile_expression = 0; size_t max_temporary_columns = 0; size_t max_temporary_non_const_columns = 0; - static ExpressionActionsSettings fromSettings(const Settings & from); - static ExpressionActionsSettings fromContext(ContextPtr from); + CompileExpressions compile_expressions = CompileExpressions::no; + + static ExpressionActionsSettings fromSettings(const Settings & from, CompileExpressions compile_expressions = CompileExpressions::no); + static ExpressionActionsSettings fromContext(ContextPtr from, CompileExpressions compile_expressions = CompileExpressions::no); }; } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 69af6a30e1d..766b0523c81 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1028,7 +1028,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain auto actions_dag = std::make_shared(columns_after_join); getRootActions(child, only_types, actions_dag); group_by_elements_actions.emplace_back( - std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(getContext()), CompileExpressions::yes)); + std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes))); } } @@ -1187,7 +1187,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai auto actions_dag = std::make_shared(columns_after_join); getRootActions(child, only_types, actions_dag); order_by_elements_actions.emplace_back( - std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(getContext()), CompileExpressions::yes)); + std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes))); } } @@ -1348,7 +1348,7 @@ ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool project_r ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool project_result, CompileExpressions compile_expressions) { return std::make_shared( - getActionsDAG(add_aliases, project_result), ExpressionActionsSettings::fromContext(getContext()), compile_expressions); + getActionsDAG(add_aliases, project_result), ExpressionActionsSettings::fromContext(getContext(), compile_expressions)); } ExpressionActionsPtr ExpressionAnalyzer::getConstActions(const ColumnsWithTypeAndName & constant_inputs) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index eed4dae1e67..225bf9ec651 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -325,7 +325,7 @@ BlockIO InterpreterInsertQuery::execute() res.pipeline.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Position); - auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(getContext()), CompileExpressions::yes); + auto actions = std::make_shared(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)); res.pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr { diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 7ecae1d193f..875953beb47 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1897,17 +1897,17 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (prewhere_info) { - auto actions_settings = ExpressionActionsSettings::fromContext(context); + auto actions_settings = ExpressionActionsSettings::fromContext(context, CompileExpressions::yes); query_info.prewhere_info = std::make_shared(); - query_info.prewhere_info->prewhere_actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings, CompileExpressions::yes); + query_info.prewhere_info->prewhere_actions = std::make_shared(prewhere_info->prewhere_actions, actions_settings); if (prewhere_info->row_level_filter_actions) - query_info.prewhere_info->row_level_filter = std::make_shared(prewhere_info->row_level_filter_actions, actions_settings, CompileExpressions::yes); + query_info.prewhere_info->row_level_filter = std::make_shared(prewhere_info->row_level_filter_actions, actions_settings); if (prewhere_info->alias_actions) - query_info.prewhere_info->alias_actions = std::make_shared(prewhere_info->alias_actions, actions_settings, CompileExpressions::yes); + query_info.prewhere_info->alias_actions = std::make_shared(prewhere_info->alias_actions, actions_settings); if (prewhere_info->remove_columns_actions) - query_info.prewhere_info->remove_columns_actions = std::make_shared(prewhere_info->remove_columns_actions, actions_settings, CompileExpressions::yes); + query_info.prewhere_info->remove_columns_actions = std::make_shared(prewhere_info->remove_columns_actions, actions_settings); query_info.prewhere_info->prewhere_column_name = prewhere_info->prewhere_column_name; query_info.prewhere_info->remove_prewhere_column = prewhere_info->remove_prewhere_column; diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index 4aa47c4a06c..fa9ea298319 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -60,7 +60,8 @@ void ArrayJoinStep::transformPipeline(QueryPipeline & pipeline, const BuildQuery pipeline.getHeader().getColumnsWithTypeAndName(), res_header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); - auto actions = std::make_shared(actions_dag, settings.getActionsSettings(), CompileExpressions::yes); + + auto actions = std::make_shared(actions_dag, settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header) { diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp index 9691da4a362..2480673d65e 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.cpp @@ -9,7 +9,7 @@ namespace DB BuildQueryPipelineSettings BuildQueryPipelineSettings::fromSettings(const Settings & from) { BuildQueryPipelineSettings settings; - settings.actions_settings = ExpressionActionsSettings::fromSettings(from); + settings.actions_settings = ExpressionActionsSettings::fromSettings(from, CompileExpressions::yes); return settings; } diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index b615d5b395c..656dcd46fe9 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -54,7 +54,7 @@ void ExpressionStep::updateInputStream(DataStream input_stream, bool keep_header void ExpressionStep::transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings & settings) { - auto expression = std::make_shared(actions_dag, settings.getActionsSettings(), CompileExpressions::yes); + auto expression = std::make_shared(actions_dag, settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header) { @@ -67,7 +67,7 @@ void ExpressionStep::transformPipeline(QueryPipeline & pipeline, const BuildQuer pipeline.getHeader().getColumnsWithTypeAndName(), output_stream->header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); - auto convert_actions = std::make_shared(convert_actions_dag, settings.getActionsSettings(), CompileExpressions::yes); + auto convert_actions = std::make_shared(convert_actions_dag, settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header) { diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index a9183a58a90..15fd5c7b673 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -67,7 +67,7 @@ void FilterStep::updateInputStream(DataStream input_stream, bool keep_header) void FilterStep::transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings & settings) { - auto expression = std::make_shared(actions_dag, settings.getActionsSettings(), CompileExpressions::yes); + auto expression = std::make_shared(actions_dag, settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header, QueryPipeline::StreamType stream_type) { @@ -81,7 +81,7 @@ void FilterStep::transformPipeline(QueryPipeline & pipeline, const BuildQueryPip pipeline.getHeader().getColumnsWithTypeAndName(), output_stream->header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); - auto convert_actions = std::make_shared(convert_actions_dag, settings.getActionsSettings(), CompileExpressions::yes); + auto convert_actions = std::make_shared(convert_actions_dag, settings.getActionsSettings()); pipeline.addSimpleTransform([&](const Block & header) { diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 6464f62b188..db82538d5a0 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -51,7 +51,7 @@ TotalsHavingStep::TotalsHavingStep( void TotalsHavingStep::transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings & settings) { - auto expression_actions = actions_dag ? std::make_shared(actions_dag, settings.getActionsSettings(), CompileExpressions::yes) : nullptr; + auto expression_actions = actions_dag ? std::make_shared(actions_dag, settings.getActionsSettings()) : nullptr; auto totals_having = std::make_shared( pipeline.getHeader(), diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 951058bd991..5a84e0c3901 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -415,8 +415,7 @@ Pipe StorageMerge::createSources( auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column)); auto adding_column_actions = std::make_shared( std::move(adding_column_dag), - ExpressionActionsSettings::fromContext(modified_context), - CompileExpressions::yes); + ExpressionActionsSettings::fromContext(modified_context, CompileExpressions::yes)); pipe.addSimpleTransform([&](const Block & stream_header) { @@ -560,7 +559,7 @@ void StorageMerge::convertingSourceStream( pipe.getHeader().getColumnsWithTypeAndName(), header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); - auto convert_actions = std::make_shared(convert_actions_dag, ExpressionActionsSettings::fromContext(local_context), CompileExpressions::yes); + auto convert_actions = std::make_shared(convert_actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes)); pipe.addSimpleTransform([&](const Block & stream_header) { diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 78314857212..89d6ffce07b 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -96,8 +96,7 @@ public: ActionsDAG::MatchColumnsMode::Name); auto convert_actions = std::make_shared( convert_actions_dag, - ExpressionActionsSettings::fromSettings(context->getSettingsRef()), - CompileExpressions::yes); + ExpressionActionsSettings::fromSettings(context->getSettingsRef(), CompileExpressions::yes)); pipe.addSimpleTransform([&](const Block & header) { From d6e357d78ce6d66036be4df5f06948b5a53f75dd Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 19 May 2021 17:35:23 +0300 Subject: [PATCH 154/202] consolidate connection loss handling in fuzzer --- programs/client/Client.cpp | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index ccf92ebc419..7973fc05205 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1366,6 +1366,27 @@ private: { const auto * exception = server_exception ? server_exception.get() : client_exception.get(); fmt::print(stderr, "Error on processing query '{}': {}\n", ast_to_process->formatForErrorMessage(), exception->message()); + + // Try to reconnect after errors, for two reasons: + // 1. We might not have realized that the server died, e.g. if + // it sent us a trace and closed connection properly. + // 2. The connection might have gotten into a wrong state and + // the next query will get false positive about + // "Unknown packet from server". + try + { + connection->forceConnected(connection_parameters.timeouts); + } + catch (...) + { + // Just report it, we'll terminate below. + fmt::print(stderr, + "Error while reconnecting to the server: Code: {}: {}\n", + getCurrentExceptionCode(), + getCurrentExceptionMessage(true)); + + assert(!connection->isConnected()); + } } if (!connection->isConnected()) @@ -1469,11 +1490,6 @@ private: server_exception.reset(); client_exception.reset(); have_error = false; - - // We have to reinitialize connection after errors, because it - // might have gotten into a wrong state and we'll get false - // positives about "Unknown packet from server". - connection->forceConnected(connection_parameters.timeouts); } else if (ast_to_process->formatForErrorMessage().size() > 500) { From 9de73997379a2e3200cfd8e6374ca124186e55b1 Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 19 May 2021 13:32:53 +0000 Subject: [PATCH 155/202] update fix test --- .../AggregateFunctionRangeSum.cpp | 56 ------------- .../AggregateFunctionSegmentLengthSum.cpp | 63 +++++++++++++++ ....h => AggregateFunctionSegmentLengthSum.h} | 79 ++++++++++--------- src/AggregateFunctions/Helpers.h | 17 ++-- .../registerAggregateFunctions.cpp | 4 +- .../01866_aggregate_function_range_sum.sql | 21 ----- ...ate_function_segment_length_sum.reference} | 0 ..._aggregate_function_segment_length_sum.sql | 21 +++++ 8 files changed, 138 insertions(+), 123 deletions(-) delete mode 100644 src/AggregateFunctions/AggregateFunctionRangeSum.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionSegmentLengthSum.cpp rename src/AggregateFunctions/{AggregateFunctionRangeSum.h => AggregateFunctionSegmentLengthSum.h} (61%) delete mode 100644 tests/queries/0_stateless/01866_aggregate_function_range_sum.sql rename tests/queries/0_stateless/{01866_aggregate_function_range_sum.reference => 01866_aggregate_function_segment_length_sum.reference} (100%) create mode 100644 tests/queries/0_stateless/01866_aggregate_function_segment_length_sum.sql diff --git a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp b/src/AggregateFunctions/AggregateFunctionRangeSum.cpp deleted file mode 100644 index 0c1cfd289a6..00000000000 --- a/src/AggregateFunctions/AggregateFunctionRangeSum.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - -namespace -{ - template