From 0762e1a8901c9cdc798582cb7f3fa74eb02b1834 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 29 Mar 2021 02:23:20 +0400 Subject: [PATCH 1/6] Implement config parsing and actual support for user_dn_detection section and user_dn placeholder substitution Refactor some config parsing code Rename some arguments to better reflect their meaning Add documentation for user_dn_detection section and user_dn placeholder in config.xml and in docs --- .../external-authenticators/ldap.md | 42 +++++++- programs/server/config.xml | 47 +++++++- src/Access/ExternalAuthenticators.cpp | 101 +++++++++++++----- src/Access/ExternalAuthenticators.h | 6 +- src/Access/LDAPAccessStorage.cpp | 29 +---- src/Access/LDAPAccessStorage.h | 4 +- src/Access/LDAPClient.cpp | 92 ++++++++++++---- src/Access/LDAPClient.h | 17 ++- 8 files changed, 252 insertions(+), 86 deletions(-) diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 1b65ecc968b..805d45e1b38 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -17,6 +17,7 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`. + localhost 636 @@ -31,6 +32,18 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`. /path/to/tls_ca_cert_dir ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + + + localhost + 389 + EXAMPLE\{user_name} + + CN=Users,DC=example,DC=com + (&(objectClass=user)(sAMAccountName={user_name})) + + no + ``` @@ -43,6 +56,15 @@ Note, that you can define multiple LDAP servers inside the `ldap_servers` sectio - `port` — LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise. - `bind_dn` — Template used to construct the DN to bind to. - The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt. +- `user_dn_detection` - Section with LDAP search parameters for detecting the actual user DN of the bound user. + - This is mainly used in search filters for further role mapping when the server is Active Directory. The resulting user DN will be used when replacing `{user_dn}` substrings wherever they are allowed. By default, user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected user DN value. + - `base_dn` - Template used to construct the base DN for the LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during the LDAP search. + - `scope` - Scope of the LDAP search. + - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). + - `search_filter` - Template used to construct the search filter for the LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. + - Note, that the special characters must be escaped properly in XML. - `verification_cooldown` — A period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request. - `enable_tls` — A flag to trigger the use of the secure connection to the LDAP server. @@ -107,7 +129,7 @@ Goes into `config.xml`. - + my_ldap_server @@ -122,6 +144,18 @@ Goes into `config.xml`. clickhouse_ + + + + my_ad_server + + CN=Users,DC=example,DC=com + CN + subtree + (&(objectClass=group)(member={user_dn})) + clickhouse_ + + ``` @@ -137,13 +171,13 @@ Note that `my_ldap_server` referred in the `ldap` section inside the `user_direc - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged-in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied. - `base_dn` — Template used to construct the base DN for the LDAP search. - - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search. + - The resulting DN will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. - `scope` — Scope of the LDAP search. - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default). - `search_filter` — Template used to construct the search filter for the LDAP search. - - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}` and `{base_dn}` substrings of the template with the actual user name, bind DN and base DN during each LDAP search. + - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, `{user_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, user DN, and base DN during each LDAP search. - Note, that the special characters must be escaped properly in XML. - - `attribute` — Attribute name whose values will be returned by the LDAP search. + - `attribute` — Attribute name whose values will be returned by the LDAP search. `cn`, by default. - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. The prefix will be removed from the original strings and the resulting strings will be treated as local role names. Empty by default. [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap/) diff --git a/programs/server/config.xml b/programs/server/config.xml index 4220ecbcacd..b6df1c42cc0 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -328,6 +328,20 @@ bind_dn - template used to construct the DN to bind to. The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual user name during each authentication attempt. + user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user. + This is mainly used in search filters for further role mapping when the server is Active Directory. The + resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default, + user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected + user DN value. + base_dn - template used to construct the base DN for the LDAP search. + The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings + of the template with the actual user name and bind DN during the LDAP search. + scope - scope of the LDAP search. + Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). + search_filter - template used to construct the search filter for the LDAP search. + The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' + substrings of the template with the actual user name, bind DN, and base DN during the LDAP search. + Note, that the special characters must be escaped properly in XML. verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server. Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request. @@ -359,6 +373,17 @@ /path/to/tls_ca_cert_dir ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + Example (typical Active Directory with configured user DN detection for further role mapping): + + localhost + 389 + EXAMPLE\{user_name} + + CN=Users,DC=example,DC=com + (&(objectClass=user)(sAMAccountName={user_name})) + + no + --> @@ -410,15 +435,16 @@ There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be applied. base_dn - template used to construct the base DN for the LDAP search. - The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings - of the template with the actual user name and bind DN during each LDAP search. + The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}' + substrings of the template with the actual user name, bind DN, and user DN during each LDAP search. scope - scope of the LDAP search. Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default). search_filter - template used to construct the search filter for the LDAP search. - The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}' - substrings of the template with the actual user name, bind DN, and base DN during each LDAP search. + The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and + '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during + each LDAP search. Note, that the special characters must be escaped properly in XML. - attribute - attribute name whose values will be returned by the LDAP search. + attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default. prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. @@ -437,6 +463,17 @@ clickhouse_ + Example (typical Active Directory with role mapping that relies on the detected user DN): + + my_ad_server + + CN=Users,DC=example,DC=com + CN + subtree + (&(objectClass=group)(member={user_dn})) + clickhouse_ + + --> diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 1cade973724..9eaf2a4b04b 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -20,13 +20,42 @@ namespace ErrorCodes namespace { -auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const String & name) +void parseLDAPSearchParams(LDAPClient::SearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) +{ + const bool has_base_dn = config.has(prefix + ".base_dn"); + const bool has_search_filter = config.has(prefix + ".search_filter"); + const bool has_attribute = config.has(prefix + ".attribute"); + const bool has_scope = config.has(prefix + ".scope"); + + if (has_base_dn) + params.base_dn = config.getString(prefix + ".base_dn"); + + if (has_search_filter) + params.search_filter = config.getString(prefix + ".search_filter"); + + if (has_attribute) + params.attribute = config.getString(prefix + ".attribute"); + + if (has_scope) + { + auto scope = config.getString(prefix + ".scope"); + boost::algorithm::to_lower(scope); + + if (scope == "base") params.scope = LDAPClient::SearchParams::Scope::BASE; + else if (scope == "one_level") params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL; + else if (scope == "subtree") params.scope = LDAPClient::SearchParams::Scope::SUBTREE; + else if (scope == "children") params.scope = LDAPClient::SearchParams::Scope::CHILDREN; + else + throw Exception("Invalid value for 'scope' field of LDAP search parameters in '" + prefix + + "' section, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS); + } +} + +void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConfiguration & config, const String & name) { if (name.empty()) throw Exception("LDAP server name cannot be empty", ErrorCodes::BAD_ARGUMENTS); - LDAPClient::Params params; - const String ldap_server_config = "ldap_servers." + name; const bool has_host = config.has(ldap_server_config + ".host"); @@ -34,6 +63,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str const bool has_bind_dn = config.has(ldap_server_config + ".bind_dn"); const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix"); const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix"); + const bool has_user_dn_detection = config.has(ldap_server_config + ".user_dn_detection"); const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown"); const bool has_enable_tls = config.has(ldap_server_config + ".enable_tls"); const bool has_tls_minimum_protocol_version = config.has(ldap_server_config + ".tls_minimum_protocol_version"); @@ -66,6 +96,14 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str params.bind_dn = auth_dn_prefix + "{user_name}" + auth_dn_suffix; } + if (has_user_dn_detection) + { + if (!params.user_dn_detection) + params.user_dn_detection = { .attribute = "dn" }; + + parseLDAPSearchParams(*params.user_dn_detection, config, ldap_server_config + ".user_dn_detection"); + } + if (has_verification_cooldown) params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")}; @@ -143,14 +181,10 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str } else params.port = (params.enable_tls == LDAPClient::Params::TLSEnable::YES ? 636 : 389); - - return params; } -auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config) +void parseKerberosParams(GSSAcceptorContext::Params & params, const Poco::Util::AbstractConfiguration & config) { - GSSAcceptorContext::Params params; - Poco::Util::AbstractConfiguration::Keys keys; config.keys("kerberos", keys); @@ -180,12 +214,20 @@ auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config) params.realm = config.getString("kerberos.realm", ""); params.principal = config.getString("kerberos.principal", ""); - - return params; } } +void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix) +{ + parseLDAPSearchParams(params, config, prefix); + + const bool has_prefix = config.has(prefix + ".prefix"); + + if (has_prefix) + params.prefix = config.getString(prefix + ".prefix"); +} + void ExternalAuthenticators::reset() { std::scoped_lock lock(mutex); @@ -229,7 +271,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur { try { - ldap_client_params_blueprint.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name)); + ldap_client_params_blueprint.erase(ldap_server_name); + parseLDAPServer(ldap_client_params_blueprint.emplace(ldap_server_name, LDAPClient::Params{}).first->second, config, ldap_server_name); } catch (...) { @@ -240,7 +283,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur try { if (kerberos_keys_count > 0) - kerberos_params = parseKerberosParams(config); + parseKerberosParams(kerberos_params.emplace(), config); } catch (...) { @@ -249,7 +292,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur } bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const BasicCredentials & credentials, - const LDAPClient::SearchParamsList * search_params, LDAPClient::SearchResultsList * search_results) const + const LDAPClient::RoleSearchParamsList * role_search_params, LDAPClient::SearchResultsList * role_search_results) const { std::optional params; std::size_t params_hash = 0; @@ -267,9 +310,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B params->password = credentials.getPassword(); params->combineCoreHash(params_hash); - if (search_params) + if (role_search_params) { - for (const auto & params_instance : *search_params) + for (const auto & params_instance : *role_search_params) { params_instance.combineHash(params_hash); } @@ -301,14 +344,14 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B // Ensure that search_params are compatible. ( - search_params == nullptr ? - entry.last_successful_search_results.empty() : - search_params->size() == entry.last_successful_search_results.size() + role_search_params == nullptr ? + entry.last_successful_role_search_results.empty() : + role_search_params->size() == entry.last_successful_role_search_results.size() ) ) { - if (search_results) - *search_results = entry.last_successful_search_results; + if (role_search_results) + *role_search_results = entry.last_successful_role_search_results; return true; } @@ -326,7 +369,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B } LDAPSimpleAuthClient client(params.value()); - const auto result = client.authenticate(search_params, search_results); + const auto result = client.authenticate(role_search_params, role_search_results); const auto current_check_timestamp = std::chrono::steady_clock::now(); // Update the cache, but only if this is the latest check and the server is still configured in a compatible way. @@ -345,9 +388,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B std::size_t new_params_hash = 0; new_params.combineCoreHash(new_params_hash); - if (search_params) + if (role_search_params) { - for (const auto & params_instance : *search_params) + for (const auto & params_instance : *role_search_params) { params_instance.combineHash(new_params_hash); } @@ -363,17 +406,17 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B entry.last_successful_params_hash = params_hash; entry.last_successful_authentication_timestamp = current_check_timestamp; - if (search_results) - entry.last_successful_search_results = *search_results; + if (role_search_results) + entry.last_successful_role_search_results = *role_search_results; else - entry.last_successful_search_results.clear(); + entry.last_successful_role_search_results.clear(); } else if ( entry.last_successful_params_hash != params_hash || ( - search_params == nullptr ? - !entry.last_successful_search_results.empty() : - search_params->size() != entry.last_successful_search_results.size() + role_search_params == nullptr ? + !entry.last_successful_role_search_results.empty() : + role_search_params->size() != entry.last_successful_role_search_results.size() ) ) { diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index c8feea7eada..24f1f7b6528 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -34,7 +34,7 @@ public: // The name and readiness of the credentials must be verified before calling these. bool checkLDAPCredentials(const String & server, const BasicCredentials & credentials, - const LDAPClient::SearchParamsList * search_params = nullptr, LDAPClient::SearchResultsList * search_results = nullptr) const; + const LDAPClient::RoleSearchParamsList * role_search_params = nullptr, LDAPClient::SearchResultsList * role_search_results = nullptr) const; bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; GSSAcceptorContext::Params getKerberosParams() const; @@ -44,7 +44,7 @@ private: { std::size_t last_successful_params_hash = 0; std::chrono::steady_clock::time_point last_successful_authentication_timestamp; - LDAPClient::SearchResultsList last_successful_search_results; + LDAPClient::SearchResultsList last_successful_role_search_results; }; using LDAPCache = std::unordered_map; // user name -> cache entry @@ -58,4 +58,6 @@ private: std::optional kerberos_params; }; +void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix); + } diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index b47a9b3e041..c1d54e8c9aa 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -68,34 +68,15 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m common_roles_cfg.insert(role_names.begin(), role_names.end()); } - LDAPClient::SearchParamsList role_search_params_cfg; + LDAPClient::RoleSearchParamsList role_search_params_cfg; if (has_role_mapping) { Poco::Util::AbstractConfiguration::Keys all_keys; config.keys(prefix, all_keys); for (const auto & key : all_keys) { - if (key != "role_mapping" && key.find("role_mapping[") != 0) - continue; - - const String rm_prefix = prefix_str + key; - const String rm_prefix_str = rm_prefix + '.'; - role_search_params_cfg.emplace_back(); - auto & rm_params = role_search_params_cfg.back(); - - rm_params.base_dn = config.getString(rm_prefix_str + "base_dn", ""); - rm_params.search_filter = config.getString(rm_prefix_str + "search_filter", ""); - rm_params.attribute = config.getString(rm_prefix_str + "attribute", "cn"); - rm_params.prefix = config.getString(rm_prefix_str + "prefix", ""); - - auto scope = config.getString(rm_prefix_str + "scope", "subtree"); - boost::algorithm::to_lower(scope); - if (scope == "base") rm_params.scope = LDAPClient::SearchParams::Scope::BASE; - else if (scope == "one_level") rm_params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL; - else if (scope == "subtree") rm_params.scope = LDAPClient::SearchParams::Scope::SUBTREE; - else if (scope == "children") rm_params.scope = LDAPClient::SearchParams::Scope::CHILDREN; - else - throw Exception("Invalid value of 'scope' field in '" + key + "' section of LDAP user directory, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS); + if (key == "role_mapping" || key.find("role_mapping[") == 0) + parseLDAPRoleSearchParams(role_search_params_cfg.emplace_back(), config, prefix_str + key); } } @@ -364,7 +345,7 @@ std::set LDAPAccessStorage::mapExternalRolesNoLock(const LDAPClient::Sea bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials, - const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const + const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const { if (!credentials.isReady()) return false; @@ -373,7 +354,7 @@ bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const C return false; if (const auto * basic_credentials = dynamic_cast(&credentials)) - return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &search_results); + return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &role_search_results); return false; } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index ea0ab47c225..33ac9f0a914 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -68,12 +68,12 @@ private: void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const; std::set mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const; bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials, - const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const; + const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const; mutable std::recursive_mutex mutex; AccessControlManager * access_control_manager = nullptr; String ldap_server_name; - LDAPClient::SearchParamsList role_search_params; + LDAPClient::RoleSearchParamsList role_search_params; std::set common_role_names; // role name that should be granted to all users at all times mutable std::map external_role_hashes; // user name -> LDAPClient::SearchResultsList hash (most recently retrieved and processed) mutable std::map> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles) diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index 5c4b7dd8d99..78b0b7f545b 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -32,6 +32,11 @@ void LDAPClient::SearchParams::combineHash(std::size_t & seed) const boost::hash_combine(seed, static_cast(scope)); boost::hash_combine(seed, search_filter); boost::hash_combine(seed, attribute); +} + +void LDAPClient::RoleSearchParams::combineHash(std::size_t & seed) const +{ + SearchParams::combineHash(seed); boost::hash_combine(seed, prefix); } @@ -42,6 +47,9 @@ void LDAPClient::Params::combineCoreHash(std::size_t & seed) const boost::hash_combine(seed, bind_dn); boost::hash_combine(seed, user); boost::hash_combine(seed, password); + + if (user_dn_detection) + user_dn_detection->combineHash(seed); } LDAPClient::LDAPClient(const Params & params_) @@ -286,18 +294,33 @@ void LDAPClient::openConnection() if (params.enable_tls == LDAPClient::Params::TLSEnable::YES_STARTTLS) diag(ldap_start_tls_s(handle, nullptr, nullptr)); + final_user_name = escapeForLDAP(params.user); + final_bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", final_user_name} }); + final_user_dn = final_bind_dn; // The default value... may be updated right after a successful bind. + switch (params.sasl_mechanism) { case LDAPClient::Params::SASLMechanism::SIMPLE: { - const auto escaped_user_name = escapeForLDAP(params.user); - const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} }); - ::berval cred; cred.bv_val = const_cast(params.password.c_str()); cred.bv_len = params.password.size(); - diag(ldap_sasl_bind_s(handle, bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr)); + diag(ldap_sasl_bind_s(handle, final_bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr)); + + // Once bound, run the user DN search query and update the default value, if asked. + if (params.user_dn_detection) + { + const auto user_dn_search_results = search(*params.user_dn_detection); + + if (user_dn_search_results.size() == 0) + throw Exception("Failed to detect user DN: empty search results", ErrorCodes::LDAP_ERROR); + + if (user_dn_search_results.size() > 1) + throw Exception("Failed to detect user DN: more than one entry in the search results", ErrorCodes::LDAP_ERROR); + + final_user_dn = *user_dn_search_results.begin(); + } break; } @@ -316,6 +339,9 @@ void LDAPClient::closeConnection() noexcept ldap_unbind_ext_s(handle, nullptr, nullptr); handle = nullptr; + final_user_name.clear(); + final_bind_dn.clear(); + final_user_dn.clear(); } LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) @@ -333,10 +359,19 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) case SearchParams::Scope::CHILDREN: scope = LDAP_SCOPE_CHILDREN; break; } - const auto escaped_user_name = escapeForLDAP(params.user); - const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} }); - const auto base_dn = replacePlaceholders(search_params.base_dn, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn} }); - const auto search_filter = replacePlaceholders(search_params.search_filter, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn}, {"{base_dn}", base_dn} }); + const auto final_base_dn = replacePlaceholders(search_params.base_dn, { + {"{user_name}", final_user_name}, + {"{bind_dn}", final_bind_dn}, + {"{user_dn}", final_user_dn} + }); + + const auto final_search_filter = replacePlaceholders(search_params.search_filter, { + {"{user_name}", final_user_name}, + {"{bind_dn}", final_bind_dn}, + {"{user_dn}", final_user_dn}, + {"{base_dn}", final_base_dn} + }); + char * attrs[] = { const_cast(search_params.attribute.c_str()), nullptr }; ::timeval timeout = { params.search_timeout.count(), 0 }; LDAPMessage* msgs = nullptr; @@ -349,7 +384,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) } }); - diag(ldap_search_ext_s(handle, base_dn.c_str(), scope, search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs)); + diag(ldap_search_ext_s(handle, final_base_dn.c_str(), scope, final_search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs)); for ( auto * msg = ldap_first_message(handle, msgs); @@ -361,6 +396,27 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) { case LDAP_RES_SEARCH_ENTRY: { + // Extract DN separately, if the requested attribute is DN. + if (boost::iequals("dn", search_params.attribute)) + { + BerElement * ber = nullptr; + + SCOPE_EXIT({ + if (ber) + { + ber_free(ber, 0); + ber = nullptr; + } + }); + + ::berval bv; + + diag(ldap_get_dn_ber(handle, msg, &ber, &bv)); + + if (bv.bv_val && bv.bv_len > 0) + result.emplace(bv.bv_val, bv.bv_len); + } + BerElement * ber = nullptr; SCOPE_EXIT({ @@ -471,12 +527,12 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) return result; } -bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, SearchResultsList * search_results) +bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results) { if (params.user.empty()) throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); - if (!search_params != !search_results) + if (!role_search_params != !role_search_results) throw Exception("Cannot return LDAP search results", ErrorCodes::BAD_ARGUMENTS); // Silently reject authentication attempt if the password is empty as if it didn't match. @@ -489,21 +545,21 @@ bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, openConnection(); // While connected, run search queries and save the results, if asked. - if (search_params) + if (role_search_params) { - search_results->clear(); - search_results->reserve(search_params->size()); + role_search_results->clear(); + role_search_results->reserve(role_search_params->size()); try { - for (const auto & single_search_params : *search_params) + for (const auto & params_instance : *role_search_params) { - search_results->emplace_back(search(single_search_params)); + role_search_results->emplace_back(search(params_instance)); } } catch (...) { - search_results->clear(); + role_search_results->clear(); throw; } } @@ -532,7 +588,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams &) throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } -bool LDAPSimpleAuthClient::authenticate(const SearchParamsList *, SearchResultsList *) +bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList *, SearchResultsList *) { throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index 4fc97bb957b..388e7ad0f0d 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -38,12 +38,20 @@ public: Scope scope = Scope::SUBTREE; String search_filter; String attribute = "cn"; + + void combineHash(std::size_t & seed) const; + }; + + struct RoleSearchParams + : public SearchParams + { String prefix; void combineHash(std::size_t & seed) const; }; - using SearchParamsList = std::vector; + using RoleSearchParamsList = std::vector; + using SearchResults = std::set; using SearchResultsList = std::vector; @@ -105,6 +113,8 @@ public: String user; String password; + std::optional user_dn_detection; + std::chrono::seconds verification_cooldown{0}; std::chrono::seconds operation_timeout{40}; @@ -134,6 +144,9 @@ protected: #if USE_LDAP LDAP * handle = nullptr; #endif + String final_user_name; + String final_bind_dn; + String final_user_dn; }; class LDAPSimpleAuthClient @@ -141,7 +154,7 @@ class LDAPSimpleAuthClient { public: using LDAPClient::LDAPClient; - bool authenticate(const SearchParamsList * search_params, SearchResultsList * search_results); + bool authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results); }; } From a9e5532da62873ae7d9920086ca83aaae161df43 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 29 Mar 2021 12:27:16 +0400 Subject: [PATCH 2/6] Fix builds: soothe the linters --- src/Access/ExternalAuthenticators.cpp | 5 ++++- src/Access/LDAPClient.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 9eaf2a4b04b..99a3347b0de 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -99,7 +99,10 @@ void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConf if (has_user_dn_detection) { if (!params.user_dn_detection) - params.user_dn_detection = { .attribute = "dn" }; + { + params.user_dn_detection.emplace(); + params.user_dn_detection->attribute = "dn"; + } parseLDAPSearchParams(*params.user_dn_detection, config, ldap_server_config + ".user_dn_detection"); } diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index 78b0b7f545b..a8f9675774b 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -313,7 +313,7 @@ void LDAPClient::openConnection() { const auto user_dn_search_results = search(*params.user_dn_detection); - if (user_dn_search_results.size() == 0) + if (user_dn_search_results.empty()) throw Exception("Failed to detect user DN: empty search results", ErrorCodes::LDAP_ERROR); if (user_dn_search_results.size() > 1) From 0e5c58c8b20d56eded6e2b786f8ae97ebd2ae466 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 19 Apr 2021 19:00:30 -0400 Subject: [PATCH 3/6] Adding user DN detection tests. --- .../testflows/ldap/role_mapping/regression.py | 3 + .../role_mapping/requirements/requirements.md | 70 ++- .../role_mapping/requirements/requirements.py | 184 ++++++- .../ldap/role_mapping/tests/common.py | 7 +- .../role_mapping/tests/user_dn_detection.py | 474 ++++++++++++++++++ 5 files changed, 728 insertions(+), 10 deletions(-) create mode 100644 tests/testflows/ldap/role_mapping/tests/user_dn_detection.py diff --git a/tests/testflows/ldap/role_mapping/regression.py b/tests/testflows/ldap/role_mapping/regression.py index 7afb6c98713..c853316ecec 100755 --- a/tests/testflows/ldap/role_mapping/regression.py +++ b/tests/testflows/ldap/role_mapping/regression.py @@ -11,6 +11,8 @@ from ldap.role_mapping.requirements import * # Cross-outs of known fails xfails = { "mapping/roles removed and added in parallel": + [(Fail, "known bug")], + "user dn detection/mapping/roles removed and added in parallel": [(Fail, "known bug")] } @@ -42,6 +44,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): Scenario(run=load("ldap.authentication.tests.sanity", "scenario"), name="ldap sanity") Feature(run=load("ldap.role_mapping.tests.server_config", "feature")) Feature(run=load("ldap.role_mapping.tests.mapping", "feature")) + Feature(run=load("ldap.role_mapping.tests.user_dn_detection", "feature")) if main(): regression() diff --git a/tests/testflows/ldap/role_mapping/requirements/requirements.md b/tests/testflows/ldap/role_mapping/requirements/requirements.md index e79baa9cd7c..fbd772b9d29 100644 --- a/tests/testflows/ldap/role_mapping/requirements/requirements.md +++ b/tests/testflows/ldap/role_mapping/requirements/requirements.md @@ -44,6 +44,11 @@ * 4.7.1 [BindDN Parameter](#binddn-parameter) * 4.7.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN](#rqsrs-014ldaprolemappingconfigurationserverbinddn) * 4.7.1.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN](#rqsrs-014ldaprolemappingconfigurationserverbinddnconflictwithauthdn) + * 4.7.2 [User DN Detection](#user-dn-detection) + * 4.7.2.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection](#rqsrs-014ldaprolemappingconfigurationserveruserdndetection) + * 4.7.2.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionbasedn) + * 4.7.2.3 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionscope) + * 4.7.2.4 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionsearchfilter) * 4.8 [External User Directory Configuration](#external-user-directory-configuration) * 4.8.1 [Syntax](#syntax) * 4.8.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingsyntax) @@ -318,6 +323,67 @@ version: 1.0 [ClickHouse] SHALL return an error if both `` and `` or `` parameters are specified as part of [LDAP] server description in the `` section of the `config.xml`. +#### User DN Detection + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection +version: 1.0 + +[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section +of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN +version: 1.0 + +[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify how +to construct the base DN for the LDAP search to detect the actual user DN. + +For example, + +```xml + + ... + CN=Users,DC=example,DC=com + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope +version: 1.0 + +[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL the scope of the +LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values + +* `base` +* `one_level` +* `children` +* `subtree` + +For example, + +```xml + + ... + one_level + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter +version: 1.0 + +[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify the LDAP search +filter used to detect the actual user DN. + +For example, + +```xml + + ... + (&(objectClass=user)(sAMAccountName={user_name})) + +``` + ### External User Directory Configuration #### Syntax @@ -382,7 +448,7 @@ version: 1.0 [ClickHouse] SHALL support the `` parameter in the `` section of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search. -The resulting `DN` SHALL be constructed by replacing all the `{user_name}` and `{bind_dn}` substrings of +The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of the template with the actual user name and bind `DN` during each [LDAP] search. #### Attribute Parameter @@ -445,7 +511,7 @@ version: 1.0 section of the `config.xml` that SHALL specify the template used to construct the [LDAP filter](https://ldap.com/ldap-filters/) for the search. -The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings +The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search. #### Prefix Parameter diff --git a/tests/testflows/ldap/role_mapping/requirements/requirements.py b/tests/testflows/ldap/role_mapping/requirements/requirements.py index b2748762e03..68ce4f5913e 100644 --- a/tests/testflows/ldap/role_mapping/requirements/requirements.py +++ b/tests/testflows/ldap/role_mapping/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.210129.1222545. +# document by TestFlows v1.6.210412.1213859. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -488,6 +488,105 @@ RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN_ConflictWith_AuthDN = Re level=4, num='4.7.1.2') +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section\n' + 'of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. \n' + '\n' + ), + link=None, + level=4, + num='4.7.2.1') + +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the \n' + '`` section of the `config.xml` that SHALL specify how \n' + 'to construct the base DN for the LDAP search to detect the actual user DN.\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + '\n' + ' ...\n' + ' CN=Users,DC=example,DC=com\n' + '\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='4.7.2.2') + +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the \n' + '`` section of the `config.xml` that SHALL the scope of the \n' + 'LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values\n' + '\n' + '* `base`\n' + '* `one_level`\n' + '* `children`\n' + '* `subtree`\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + '\n' + ' ...\n' + ' one_level\n' + '\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='4.7.2.3') + +RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter = Requirement( + name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the \n' + '`` section of the `config.xml` that SHALL specify the LDAP search\n' + 'filter used to detect the actual user DN.\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + '\n' + ' ...\n' + ' (&(objectClass=user)(sAMAccountName={user_name}))\n' + '\n' + '```\n' + '\n' + ), + link=None, + level=4, + num='4.7.2.4') + RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Syntax = Requirement( name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax', version='1.0', @@ -587,7 +686,7 @@ RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_BaseDN = Req '[ClickHouse] SHALL support the `` parameter in the `` section \n' 'of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search.\n' '\n' - 'The resulting `DN` SHALL be constructed by replacing all the `{user_name}` and `{bind_dn}` substrings of \n' + 'The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of \n' 'the template with the actual user name and bind `DN` during each [LDAP] search.\n' '\n' ), @@ -724,7 +823,7 @@ RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SearchFilter 'section of the `config.xml` that SHALL specify the template used to construct \n' 'the [LDAP filter](https://ldap.com/ldap-filters/) for the search.\n' '\n' - 'The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings \n' + 'The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings \n' 'of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search.\n' ' \n' ), @@ -872,6 +971,11 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( Heading(name='BindDN Parameter', level=3, num='4.7.1'), Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN', level=4, num='4.7.1.1'), Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN', level=4, num='4.7.1.2'), + Heading(name='User DN Detection', level=3, num='4.7.2'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection', level=4, num='4.7.2.1'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN', level=4, num='4.7.2.2'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope', level=4, num='4.7.2.3'), + Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter', level=4, num='4.7.2.4'), Heading(name='External User Directory Configuration', level=2, num='4.8'), Heading(name='Syntax', level=3, num='4.8.1'), Heading(name='RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax', level=4, num='4.8.1.1'), @@ -930,6 +1034,10 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( RQ_SRS_014_LDAP_RoleMapping_Authentication_Parallel_SameUser, RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN, RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_BindDN_ConflictWith_AuthDN, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope, + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_Syntax, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_SpecialCharactersEscaping, RQ_SRS_014_LDAP_RoleMapping_Configuration_UserDirectory_RoleMapping_MultipleSections, @@ -996,6 +1104,11 @@ SRS_014_ClickHouse_LDAP_Role_Mapping = Specification( * 4.7.1 [BindDN Parameter](#binddn-parameter) * 4.7.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN](#rqsrs-014ldaprolemappingconfigurationserverbinddn) * 4.7.1.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.BindDN.ConflictWith.AuthDN](#rqsrs-014ldaprolemappingconfigurationserverbinddnconflictwithauthdn) + * 4.7.2 [User DN Detection](#user-dn-detection) + * 4.7.2.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection](#rqsrs-014ldaprolemappingconfigurationserveruserdndetection) + * 4.7.2.2 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionbasedn) + * 4.7.2.3 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionscope) + * 4.7.2.4 [RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter](#rqsrs-014ldaprolemappingconfigurationserveruserdndetectionsearchfilter) * 4.8 [External User Directory Configuration](#external-user-directory-configuration) * 4.8.1 [Syntax](#syntax) * 4.8.1.1 [RQ.SRS-014.LDAP.RoleMapping.Configuration.UserDirectory.RoleMapping.Syntax](#rqsrs-014ldaprolemappingconfigurationuserdirectoryrolemappingsyntax) @@ -1270,6 +1383,67 @@ version: 1.0 [ClickHouse] SHALL return an error if both `` and `` or `` parameters are specified as part of [LDAP] server description in the `` section of the `config.xml`. +#### User DN Detection + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection +version: 1.0 + +[ClickHouse] SHALL support the `user_dn_detection` sub-section in the `` section +of the `config.xml` that SHALL be used to enable detecting the actual user DN of the bound user. + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.BaseDN +version: 1.0 + +[ClickHouse] SHALL support `base_dn` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify how +to construct the base DN for the LDAP search to detect the actual user DN. + +For example, + +```xml + + ... + CN=Users,DC=example,DC=com + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.Scope +version: 1.0 + +[ClickHouse] SHALL support `scope` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL the scope of the +LDAP search to detect the actual user DN. The `scope` parameter SHALL support the following values + +* `base` +* `one_level` +* `children` +* `subtree` + +For example, + +```xml + + ... + one_level + +``` + +##### RQ.SRS-014.LDAP.RoleMapping.Configuration.Server.UserDNDetection.SearchFilter +version: 1.0 + +[ClickHouse] SHALL support `search_filter` parameter in the `user_dn_detection` sub-section in the +`` section of the `config.xml` that SHALL specify the LDAP search +filter used to detect the actual user DN. + +For example, + +```xml + + ... + (&(objectClass=user)(sAMAccountName={user_name})) + +``` + ### External User Directory Configuration #### Syntax @@ -1334,7 +1508,7 @@ version: 1.0 [ClickHouse] SHALL support the `` parameter in the `` section of the `config.xml` that SHALL specify the template to be used to construct the base `DN` for the [LDAP] search. -The resulting `DN` SHALL be constructed by replacing all the `{user_name}` and `{bind_dn}` substrings of +The resulting `DN` SHALL be constructed by replacing all the `{user_name}`, `{bind_dn}`, and `user_dn` substrings of the template with the actual user name and bind `DN` during each [LDAP] search. #### Attribute Parameter @@ -1397,7 +1571,7 @@ version: 1.0 section of the `config.xml` that SHALL specify the template used to construct the [LDAP filter](https://ldap.com/ldap-filters/) for the search. -The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings +The resulting filter SHALL be constructed by replacing all `{user_name}`, `{bind_dn}`, `{base_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind `DN`, and base `DN` during each the [LDAP] search. #### Prefix Parameter diff --git a/tests/testflows/ldap/role_mapping/tests/common.py b/tests/testflows/ldap/role_mapping/tests/common.py index 33ad4a46f52..565503296e3 100644 --- a/tests/testflows/ldap/role_mapping/tests/common.py +++ b/tests/testflows/ldap/role_mapping/tests/common.py @@ -24,11 +24,12 @@ def create_table(self, name, create_statement, on_cluster=False): node.query(f"DROP TABLE IF EXISTS {name}") @TestStep(Given) -def add_ldap_servers_configuration(self, servers, config_d_dir="/etc/clickhouse-server/config.d", +def add_ldap_servers_configuration(self, servers, config=None, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", timeout=60, restart=False): """Add LDAP servers configuration to config.xml. """ - config = create_ldap_servers_config_content(servers, config_d_dir, config_file) + if config is None: + config = create_ldap_servers_config_content(servers, config_d_dir, config_file) return add_config(config, restart=restart) @TestStep(Given) @@ -249,4 +250,4 @@ def create_ldap_external_user_directory_config_content(server=None, roles=None, def create_entries_ldap_external_user_directory_config_content(entries, **kwargs): """Create LDAP external user directory configuration file content. """ - return create_xml_config_content(entries, **kwargs) \ No newline at end of file + return create_xml_config_content(entries, **kwargs) diff --git a/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py new file mode 100644 index 00000000000..9ec24040973 --- /dev/null +++ b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py @@ -0,0 +1,474 @@ +# -*- coding: utf-8 -*- +import importlib + +from testflows.core import * +from testflows.asserts import error + +from ldap.role_mapping.requirements import * +from ldap.role_mapping.tests.common import * + +@TestOutline +def check_config(self, entries, valid=True, ldap_server="openldap1", user="user1", password="user1"): + """Apply LDAP server configuration and check login. + """ + if valid: + exitcode = 0 + message = "1" + else: + exitcode = 4 + message = "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" + + with Given("I add LDAP server configuration"): + config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + add_ldap_servers_configuration(servers=None, config=config) + + with And("I add LDAP external user directory configuration"): + add_ldap_external_user_directory(server=ldap_server, + role_mappings=None, restart=True) + + with When(f"I login I try to login as an LDAP user"): + r = self.context.node.query(f"SELECT 1", settings=[ + ("user", user), ("password", password)], exitcode=exitcode, message=message) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") +) +def config_invalid_base_dn(self): + """Check when invalid `base_dn` is specified in the user_dn_detection section. + """ + + with Given("I define LDAP server configuration with invalid base_dn"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=user,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") +) +def config_empty_base_dn(self): + """Check when empty `base_dn` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid base_dn"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0") +) +def config_missing_base_dn(self): + """Check when missing `base_dn` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid base_dn"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + # FIXME +) +def config_invalid_search_filter(self): + """Check when invalid `search_filter` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid search_filter"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPersons)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_missing_search_filter(self): + """Check when missing `search_filter` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid search_filter"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_empty_search_filter(self): + """Check when empty `search_filter` is specified in the user_dn_detection section. + """ + with Given("I define LDAP server configuration with invalid search_filter"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "" + } + } + } + ] + } + + check_config(entries=entries, valid=False) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"), + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_valid(self): + """Check valid config with valid user_dn_detection section. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=True) + +@TestScenario +@Tags("config") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"), + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0") +) +def config_valid_tls_connection(self): + """Check valid config with valid user_dn_detection section when + using LDAP that is configured to use TLS connection. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "tls_require_cert": "never", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + } + ] + } + + check_config(entries=entries, valid=True, ldap_server="openldap2", user="user2", password="user2") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope("1.0") +) +@Examples("scope base_dn", [ + ("base", "cn=user1,ou=users,dc=company,dc=com"), + ("one_level","ou=users,dc=company,dc=com"), + ("children","ou=users,dc=company,dc=com"), + ("subtree","ou=users,dc=company,dc=com") # default value +]) +def check_valid_scope_values(self, scope, base_dn): + """Check configuration with valid scope values. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": base_dn, + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + "scope": scope + } + } + } + ] + } + + check_config(entries=entries, valid=True) + +@TestSuite +def mapping(self): + """Run all role mapping tests with both + openldap1 and openldap2 configured to use + user DN detection. + """ + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True, + "dn": "cn=user1,ou=users,dc=company,dc=com"}, + ] + + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + }, + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + "tls_require_cert": "never", + "user_dn_detection": { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))" + } + } + }, + ] + } + + with Given("I add LDAP servers configuration"): + config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + add_ldap_servers_configuration(servers=None, config=config) + + for scenario in loads(importlib.import_module("tests.mapping", package=None), Scenario): + scenario(ldap_server="openldap1", ldap_user=users[0]) + +@TestOutline +def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection): + """Check that roles get mapped properly when bind_dn and user_dn are different + by creating LDAP users that have switched uid parameter values. + """ + with Given("I define LDAP server configuration"): + entries = { + "ldap_servers": [ + { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "bind_dn": "cn={user_name},ou=users,dc=company,dc=com", + } + } + ] + } + + if user_dn_detection: + with And("I enable user dn detection"): + entries["ldap_servers"][0]["openldap1"]["user_dn_detection"] = { + "base_dn": "ou=users,dc=company,dc=com", + "search_filter": "(&(objectClass=inetOrgPerson)(uid={user_name}))", + "scope": "subtree" + } + + with And("I define role mappings"): + role_mappings = [ + { + "base_dn": "ou=groups,dc=company,dc=com", + "attribute": "cn", + "search_filter": f"(&(objectClass=groupOfUniqueNames)(uniquemember={{{map_by}}}))", + "prefix":"" + } + ] + + with Given("I add LDAP users"): + first_user = add_ldap_users(users=[ + {"cn": f"first_user", "userpassword": "user", "uid": "second_user"} + ])[0] + + second_user = add_ldap_users(users=[ + {"cn": f"second_user", "userpassword": "user", "uid": "first_user"} + ])[0] + + with Given("I add LDAP groups"): + groups = add_ldap_groups(groups=({"cn": f"role0_{uid}"}, {"cn": f"role1_{uid}"})) + + with And("I add LDAP user to each LDAP group"): + with By("adding first group to first user"): + add_user_to_group_in_ldap(user=first_user, group=groups[0]) + with And("adding second group to second user"): + add_user_to_group_in_ldap(user=second_user, group=groups[1]) + + with And("I add RBAC roles"): + roles = add_rbac_roles(roles=(f"role0_{uid}", f"role1_{uid}")) + + with Given("I add LDAP server configuration"): + config = create_xml_config_content(entries=entries, config_file="ldap_servers.xml") + add_ldap_servers_configuration(servers=None, config=config) + + with And("I add LDAP external user directory configuration"): + add_ldap_external_user_directory(server=self.context.ldap_node.name, + role_mappings=role_mappings, restart=True) + +@TestScenario +def map_roles_by_user_dn_when_base_dn_and_user_dn_are_different(self): + """Check the case when we map roles using user_dn then + the first user has uid of second user and second user + has uid of first user and configuring user DN detection to + determine user_dn based on the uid value so that user_dn + for the first user will be bind_dn of the second user and + vice versa. + """ + uid = getuid() + + setup_different_bind_dn_and_user_dn(uid=uid, map_by="user_dn", user_dn_detection=True) + + with When(f"I login as first LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "first_user"), ("password", "user")]) + + with Then("I expect the first user to have mapped LDAP roles from second user"): + assert f"GRANT role1_{uid} TO first_user" in r.output, error() + + with When(f"I login as second LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "second_user"), ("password", "user")]) + + with Then("I expect the second user to have mapped LDAP roles from first user"): + assert f"GRANT role0_{uid} TO second_user" in r.output, error() + +@TestScenario +def map_roles_by_bind_dn_when_base_dn_and_user_dn_are_different(self): + """Check the case when we map roles by bind_dn when bind_dn and user_dn + are different. + """ + uid = getuid() + + setup_different_bind_dn_and_user_dn(uid=uid, map_by="bind_dn", user_dn_detection=True) + + with When(f"I login as first LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "first_user"), ("password", "user")]) + + with Then("I expect the first user to have no mapped LDAP roles"): + assert f"GRANT role0_{uid} TO first_user" == r.output, error() + + with When(f"I login as second LDAP user"): + r = self.context.node.query(f"SHOW GRANTS", settings=[ + ("user", "second_user"), ("password", "user")]) + + with Then("I expect the second user to have no mapped LDAP roles"): + assert f"GRANT role1_{uid} TO second_user" in r.output, error() + +@TestFeature +@Name("user dn detection") +@Requirements( + RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection("1.0") +) +def feature(self): + """Check LDAP user DN detection. + """ + self.context.node = self.context.cluster.node("clickhouse1") + self.context.ldap_node = self.context.cluster.node("openldap1") + + with Given("I fix LDAP access permissions"): + fix_ldap_permissions(node=self.context.cluster.node("openldap1")) + fix_ldap_permissions(node=self.context.cluster.node("openldap2")) + + for scenario in ordered(loads(current_module(), Scenario)): + scenario() + + Suite(run=mapping) From 2b62ce904455547d2bd413a29e55ec49d6cf8b9b Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 18 May 2021 11:35:51 +0300 Subject: [PATCH 4/6] Add smoke test for local address --- src/Common/tests/gtest_local_address.cpp | 19 +++++++++++++++++++ .../tests/gtest_compressionCodec.cpp | 10 ++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 src/Common/tests/gtest_local_address.cpp diff --git a/src/Common/tests/gtest_local_address.cpp b/src/Common/tests/gtest_local_address.cpp new file mode 100644 index 00000000000..504fba19713 --- /dev/null +++ b/src/Common/tests/gtest_local_address.cpp @@ -0,0 +1,19 @@ +#include +#include +#include +#include +#include + + +TEST(LocalAddress, SmokeTest) +{ + auto cmd = DB::ShellCommand::executeDirect("/bin/hostname", {"-i"}); + std::string address_str; + DB::readString(address_str, cmd->out); + cmd->wait(); + std::cerr << "Got Address:" << address_str << std::endl; + + Poco::Net::IPAddress address(address_str); + + EXPECT_TRUE(DB::isLocalAddress(address)); +} diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index 20fe5476807..6ba2d3457ea 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -345,10 +345,12 @@ CodecTestSequence operator*(CodecTestSequence && left, T times) std::ostream & operator<<(std::ostream & ostr, const Codec & codec) { - return ostr << "Codec{" - << "name: " << codec.codec_statement - << ", expected_compression_ratio: " << *codec.expected_compression_ratio - << "}"; + ostr << "Codec{" + << "name: " << codec.codec_statement; + if (codec.expected_compression_ratio) + return ostr << ", expected_compression_ratio: " << *codec.expected_compression_ratio << "}"; + else + return ostr << "}"; } std::ostream & operator<<(std::ostream & ostr, const CodecTestSequence & seq) From 70e3ffc895b509c0c3d9c814366eccbc2c6008a2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 18 May 2021 14:46:45 +0300 Subject: [PATCH 5/6] Speedup test --- tests/integration/test_jbod_balancer/test.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py index abc6a0bff11..ef0308cc658 100644 --- a/tests/integration/test_jbod_balancer/test.py +++ b/tests/integration/test_jbod_balancer/test.py @@ -92,7 +92,10 @@ def test_jbod_balanced_merge(start_cluster): node1.query("create table tmp1 as tbl") node1.query("create table tmp2 as tbl") - for i in range(200): + p = Pool(20) + + def task(i): + print("Processing insert {}/{}".format(i, 200)) # around 1k per block node1.query( "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" @@ -104,6 +107,8 @@ def test_jbod_balanced_merge(start_cluster): "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" ) + p.map(task, range(200)) + time.sleep(1) check_balance(node1, "tbl") @@ -151,8 +156,10 @@ def test_replicated_balanced_merge_fetch(start_cluster): node.query("create table tmp2 as tmp1") node2.query("alter table tbl modify setting always_fetch_merged_part = 1") + p = Pool(20) - for i in range(200): + def task(i): + print("Processing insert {}/{}".format(i, 200)) # around 1k per block node1.query( "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" @@ -170,6 +177,8 @@ def test_replicated_balanced_merge_fetch(start_cluster): "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" ) + p.map(task, range(200)) + node2.query("SYSTEM SYNC REPLICA tbl", timeout=10) check_balance(node1, "tbl") From 829cdf3c21cfd9c2e5d56356d667d5b9346b6225 Mon Sep 17 00:00:00 2001 From: lulichao <44281662+lulichao@users.noreply.github.com> Date: Wed, 19 May 2021 10:58:47 +0800 Subject: [PATCH 6/6] Update syntax.md modify wrongly writen --- docs/zh/sql-reference/syntax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/sql-reference/syntax.md b/docs/zh/sql-reference/syntax.md index 8c331db1139..c05c5a1a7bf 100644 --- a/docs/zh/sql-reference/syntax.md +++ b/docs/zh/sql-reference/syntax.md @@ -14,7 +14,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') 含`INSERT INTO t VALUES` 的部分由完整SQL解析器处理，包含数据的部分 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 交给快速流式解析器解析。通过设置参数 [input_format_values_interpret_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions)，你也可以对数据部分开启完整SQL解析器。当 `input_format_values_interpret_expressions = 1` 时，CH优先采用快速流式解析器来解析数据。如果失败，CH再尝试用完整SQL解析器来处理，就像处理SQL [expression](#syntax-expressions) 一样。 -数据可以采用任何格式。当CH接受到请求时，服务端先在内存中计算不超过 [max_query_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。 +数据可以采用任何格式。当CH接收到请求时，服务端先在内存中计算不超过 [max_query_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。 这将避免在处理大型的 `INSERT`语句时出现问题。