Merge remote-tracking branch 'origin' into integration-2

2024-11-22 23:52:03 +00:00 · 2021-05-19 13:25:30 +03:00 · 2021-05-19 13:25:30 +03:00 · 1f4e28accb
commit 1f4e28accb
parent 7a5895815b 4c0fde9051
14 changed files with 306 additions and 108 deletions
--- a/docs/en/operations/external-authenticators/ldap.md
+++ b/docs/en/operations/external-authenticators/ldap.md
@ -17,6 +17,7 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`.
 <yandex>
    <!- ... -->
    <ldap_servers>
+        <!- Typical LDAP server. -->
        <my_ldap_server>
            <host>localhost</host>
            <port>636</port>
@ -31,6 +32,18 @@ To define LDAP server you must add `ldap_servers` section to the `config.xml`.
            <tls_ca_cert_dir>/path/to/tls_ca_cert_dir</tls_ca_cert_dir>
            <tls_cipher_suite>ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384</tls_cipher_suite>
        </my_ldap_server>
+
+        <!- Typical Active Directory with configured user DN detection for further role mapping. -->
+        <my_ad_server>
+            <host>localhost</host>
+            <port>389</port>
+            <bind_dn>EXAMPLE\{user_name}</bind_dn>
+            <user_dn_detection>
+                <base_dn>CN=Users,DC=example,DC=com</base_dn>
+                <search_filter>(&amp;(objectClass=user)(sAMAccountName={user_name}))</search_filter>
+            </user_dn_detection>
+            <enable_tls>no</enable_tls>
+        </my_ad_server>
    </ldap_servers>
 </yandex>
 ```
@ -43,6 +56,15 @@ Note, that you can define multiple LDAP servers inside the `ldap_servers` sectio
 - `port` — LDAP server port, default is `636` if `enable_tls` is set to `true`, `389` otherwise.
 - `bind_dn` — Template used to construct the DN to bind to.
    - The resulting DN will be constructed by replacing all `{user_name}` substrings of the template with the actual user name during each authentication attempt.
+- `user_dn_detection` - Section with LDAP search parameters for detecting the actual user DN of the bound user.
+    - This is mainly used in search filters for further role mapping when the server is Active Directory. The resulting user DN will be used when replacing `{user_dn}` substrings wherever they are allowed. By default, user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected user DN value.
+        - `base_dn` - Template used to construct the base DN for the LDAP search.
+            - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during the LDAP search.
+        - `scope` - Scope of the LDAP search.
+            - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default).
+        - `search_filter` - Template used to construct the search filter for the LDAP search.
+            - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, and base DN during the LDAP search.
+            - Note, that the special characters must be escaped properly in XML.
 - `verification_cooldown` — A period of time, in seconds, after a successful bind attempt, during which the user will be assumed to be successfully authenticated for all consecutive requests without contacting the LDAP server.
    - Specify `0` (the default) to disable caching and force contacting the LDAP server for each authentication request.
 - `enable_tls` — A flag to trigger the use of the secure connection to the LDAP server.
@ -107,7 +129,7 @@ Goes into `config.xml`.
 <yandex>
    <!- ... -->
    <user_directories>
-        <!- ... -->
+        <!- Typical LDAP server. -->
        <ldap>
            <server>my_ldap_server</server>
            <roles>
@ -122,6 +144,18 @@ Goes into `config.xml`.
                <prefix>clickhouse_</prefix>
            </role_mapping>
        </ldap>
+
+        <!- Typical Active Directory with role mapping that relies on the detected user DN. -->
+        <ldap>
+            <server>my_ad_server</server>
+            <role_mapping>
+                <base_dn>CN=Users,DC=example,DC=com</base_dn>
+                <attribute>CN</attribute>
+                <scope>subtree</scope>
+                <search_filter>(&amp;(objectClass=group)(member={user_dn}))</search_filter>
+                <prefix>clickhouse_</prefix>
+            </role_mapping>
+        </ldap>
    </user_directories>
 </yandex>
 ```
@ -137,13 +171,13 @@ Note that `my_ldap_server` referred in the `ldap` section inside the `user_direc
    - When a user authenticates, while still bound to LDAP, an LDAP search is performed using `search_filter` and the name of the logged-in user. For each entry found during that search, the value of the specified attribute is extracted. For each attribute value that has the specified prefix, the prefix is removed, and the rest of the value becomes the name of a local role defined in ClickHouse, which is expected to be created beforehand by the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement.
    - There can be multiple `role_mapping` sections defined inside the same `ldap` section. All of them will be applied.
        - `base_dn` — Template used to construct the base DN for the LDAP search.
-           - The resulting DN will be constructed by replacing all `{user_name}` and `{bind_dn}` substrings of the template with the actual user name and bind DN during each LDAP search.
+            - The resulting DN will be constructed by replacing all `{user_name}`, `{bind_dn}`, and `{user_dn}` substrings of the template with the actual user name, bind DN, and user DN during each LDAP search.
        - `scope` — Scope of the LDAP search.
            - Accepted values are: `base`, `one_level`, `children`, `subtree` (the default).
        - `search_filter` — Template used to construct the search filter for the LDAP search.
-            - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}` and `{base_dn}` substrings of the template with the actual user name, bind DN and base DN during each LDAP search.
+            - The resulting filter will be constructed by replacing all `{user_name}`, `{bind_dn}`, `{user_dn}`, and `{base_dn}` substrings of the template with the actual user name, bind DN, user DN, and base DN during each LDAP search.
            - Note, that the special characters must be escaped properly in XML.
-        - `attribute` — Attribute name whose values will be returned by the LDAP search.
+        - `attribute` — Attribute name whose values will be returned by the LDAP search. `cn`, by default.
        - `prefix` — Prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. The prefix will be removed from the original strings and the resulting strings will be treated as local role names. Empty by default.

 [Original article](https://clickhouse.tech/docs/en/operations/external-authenticators/ldap/) <!--hide-->
--- a/docs/zh/sql-reference/syntax.md
+++ b/docs/zh/sql-reference/syntax.md
@ -14,7 +14,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def')

 含`INSERT INTO t VALUES` 的部分由完整SQL解析器处理，包含数据的部分 `(1, 'Hello, world'), (2, 'abc'), (3, 'def')` 交给快速流式解析器解析。通过设置参数 [input_format_values_interpret_expressions](../operations/settings/settings.md#settings-input_format_values_interpret_expressions)，你也可以对数据部分开启完整SQL解析器。当 `input_format_values_interpret_expressions = 1` 时，CH优先采用快速流式解析器来解析数据。如果失败，CH再尝试用完整SQL解析器来处理，就像处理SQL [expression](#syntax-expressions) 一样。

-数据可以采用任何格式。当CH接受到请求时，服务端先在内存中计算不超过 [max_query_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。
+数据可以采用任何格式。当CH接收到请求时，服务端先在内存中计算不超过 [max_query_size](../operations/settings/settings.md#settings-max_query_size) 字节的请求数据（默认1 mb），然后剩下部分交给快速流式解析器。

 这将避免在处理大型的 `INSERT`语句时出现问题。

--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@ -362,6 +362,20 @@
                bind_dn - template used to construct the DN to bind to.
                        The resulting DN will be constructed by replacing all '{user_name}' substrings of the template with the actual
                         user name during each authentication attempt.
+                user_dn_detection - section with LDAP search parameters for detecting the actual user DN of the bound user.
+                        This is mainly used in search filters for further role mapping when the server is Active Directory. The
+                         resulting user DN will be used when replacing '{user_dn}' substrings wherever they are allowed. By default,
+                         user DN is set equal to bind DN, but once search is performed, it will be updated with to the actual detected
+                         user DN value.
+                    base_dn - template used to construct the base DN for the LDAP search.
+                            The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
+                             of the template with the actual user name and bind DN during the LDAP search.
+                    scope - scope of the LDAP search.
+                            Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
+                    search_filter - template used to construct the search filter for the LDAP search.
+                            The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
+                             substrings of the template with the actual user name, bind DN, and base DN during the LDAP search.
+                            Note, that the special characters must be escaped properly in XML.
                verification_cooldown - a period of time, in seconds, after a successful bind attempt, during which a user will be assumed
                         to be successfully authenticated for all consecutive requests without contacting the LDAP server.
                        Specify 0 (the default) to disable caching and force contacting the LDAP server for each authentication request.
@ -393,6 +407,17 @@
                    <tls_ca_cert_dir>/path/to/tls_ca_cert_dir</tls_ca_cert_dir>
                    <tls_cipher_suite>ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384</tls_cipher_suite>
                </my_ldap_server>
+             Example (typical Active Directory with configured user DN detection for further role mapping):
+                <my_ad_server>
+                    <host>localhost</host>
+                    <port>389</port>
+                    <bind_dn>EXAMPLE\{user_name}</bind_dn>
+                    <user_dn_detection>
+                        <base_dn>CN=Users,DC=example,DC=com</base_dn>
+                        <search_filter>(&amp;(objectClass=user)(sAMAccountName={user_name}))</search_filter>
+                    </user_dn_detection>
+                    <enable_tls>no</enable_tls>
+                </my_ad_server>
        -->
    </ldap_servers>

@ -444,15 +469,16 @@
                        There can be multiple 'role_mapping' sections defined inside the same 'ldap' section. All of them will be
                         applied.
                    base_dn - template used to construct the base DN for the LDAP search.
-                            The resulting DN will be constructed by replacing all '{user_name}' and '{bind_dn}' substrings
-                             of the template with the actual user name and bind DN during each LDAP search.
+                            The resulting DN will be constructed by replacing all '{user_name}', '{bind_dn}', and '{user_dn}'
+                             substrings of the template with the actual user name, bind DN, and user DN during each LDAP search.
                    scope - scope of the LDAP search.
                            Accepted values are: 'base', 'one_level', 'children', 'subtree' (the default).
                    search_filter - template used to construct the search filter for the LDAP search.
-                            The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', and '{base_dn}'
-                             substrings of the template with the actual user name, bind DN, and base DN during each LDAP search.
+                            The resulting filter will be constructed by replacing all '{user_name}', '{bind_dn}', '{user_dn}', and
+                             '{base_dn}' substrings of the template with the actual user name, bind DN, user DN, and base DN during
+                             each LDAP search.
                            Note, that the special characters must be escaped properly in XML.
-                    attribute - attribute name whose values will be returned by the LDAP search.
+                    attribute - attribute name whose values will be returned by the LDAP search. 'cn', by default.
                    prefix - prefix, that will be expected to be in front of each string in the original list of strings returned by
                             the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated
                             as local role names. Empty, by default.
@ -471,6 +497,17 @@
                        <prefix>clickhouse_</prefix>
                    </role_mapping>
                </ldap>
+             Example (typical Active Directory with role mapping that relies on the detected user DN):
+                <ldap>
+                    <server>my_ad_server</server>
+                    <role_mapping>
+                        <base_dn>CN=Users,DC=example,DC=com</base_dn>
+                        <attribute>CN</attribute>
+                        <scope>subtree</scope>
+                        <search_filter>(&amp;(objectClass=group)(member={user_dn}))</search_filter>
+                        <prefix>clickhouse_</prefix>
+                    </role_mapping>
+                </ldap>
        -->
    </user_directories>

--- a/src/Access/ExternalAuthenticators.cpp
+++ b/src/Access/ExternalAuthenticators.cpp
@ -20,13 +20,42 @@ namespace ErrorCodes
 namespace
 {

-auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const String & name)
+void parseLDAPSearchParams(LDAPClient::SearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix)
+{
+    const bool has_base_dn = config.has(prefix + ".base_dn");
+    const bool has_search_filter = config.has(prefix + ".search_filter");
+    const bool has_attribute = config.has(prefix + ".attribute");
+    const bool has_scope = config.has(prefix + ".scope");
+
+    if (has_base_dn)
+        params.base_dn = config.getString(prefix + ".base_dn");
+
+    if (has_search_filter)
+        params.search_filter = config.getString(prefix + ".search_filter");
+
+    if (has_attribute)
+        params.attribute = config.getString(prefix + ".attribute");
+
+    if (has_scope)
+    {
+        auto scope = config.getString(prefix + ".scope");
+        boost::algorithm::to_lower(scope);
+
+        if (scope == "base")           params.scope = LDAPClient::SearchParams::Scope::BASE;
+        else if (scope == "one_level") params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL;
+        else if (scope == "subtree")   params.scope = LDAPClient::SearchParams::Scope::SUBTREE;
+        else if (scope == "children")  params.scope = LDAPClient::SearchParams::Scope::CHILDREN;
+        else
+            throw Exception("Invalid value for 'scope' field of LDAP search parameters in '" + prefix +
+                "' section, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS);
+    }
+}
+
+void parseLDAPServer(LDAPClient::Params & params, const Poco::Util::AbstractConfiguration & config, const String & name)
 {
    if (name.empty())
        throw Exception("LDAP server name cannot be empty", ErrorCodes::BAD_ARGUMENTS);

-    LDAPClient::Params params;
-
    const String ldap_server_config = "ldap_servers." + name;

    const bool has_host = config.has(ldap_server_config + ".host");
@ -34,6 +63,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
    const bool has_bind_dn = config.has(ldap_server_config + ".bind_dn");
    const bool has_auth_dn_prefix = config.has(ldap_server_config + ".auth_dn_prefix");
    const bool has_auth_dn_suffix = config.has(ldap_server_config + ".auth_dn_suffix");
+    const bool has_user_dn_detection = config.has(ldap_server_config + ".user_dn_detection");
    const bool has_verification_cooldown = config.has(ldap_server_config + ".verification_cooldown");
    const bool has_enable_tls = config.has(ldap_server_config + ".enable_tls");
    const bool has_tls_minimum_protocol_version = config.has(ldap_server_config + ".tls_minimum_protocol_version");
@ -66,6 +96,17 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
        params.bind_dn = auth_dn_prefix + "{user_name}" + auth_dn_suffix;
    }

+    if (has_user_dn_detection)
+    {
+        if (!params.user_dn_detection)
+        {
+            params.user_dn_detection.emplace();
+            params.user_dn_detection->attribute = "dn";
+        }
+
+        parseLDAPSearchParams(*params.user_dn_detection, config, ldap_server_config + ".user_dn_detection");
+    }
+
    if (has_verification_cooldown)
        params.verification_cooldown = std::chrono::seconds{config.getUInt64(ldap_server_config + ".verification_cooldown")};

@ -143,14 +184,10 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
    }
    else
        params.port = (params.enable_tls == LDAPClient::Params::TLSEnable::YES ? 636 : 389);
-
-    return params;
 }

-auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config)
+void parseKerberosParams(GSSAcceptorContext::Params & params, const Poco::Util::AbstractConfiguration & config)
 {
-    GSSAcceptorContext::Params params;
-
    Poco::Util::AbstractConfiguration::Keys keys;
    config.keys("kerberos", keys);

@ -180,12 +217,20 @@ auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config)

    params.realm = config.getString("kerberos.realm", "");
    params.principal = config.getString("kerberos.principal", "");
-
-    return params;
 }

 }

+void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix)
+{
+    parseLDAPSearchParams(params, config, prefix);
+
+    const bool has_prefix = config.has(prefix + ".prefix");
+
+    if (has_prefix)
+        params.prefix = config.getString(prefix + ".prefix");
+}
+
 void ExternalAuthenticators::reset()
 {
    std::scoped_lock lock(mutex);
@ -229,7 +274,8 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
    {
        try
        {
-            ldap_client_params_blueprint.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name));
+            ldap_client_params_blueprint.erase(ldap_server_name);
+            parseLDAPServer(ldap_client_params_blueprint.emplace(ldap_server_name, LDAPClient::Params{}).first->second, config, ldap_server_name);
        }
        catch (...)
        {
@ -240,7 +286,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
    try
    {
        if (kerberos_keys_count > 0)
-            kerberos_params = parseKerberosParams(config);
+            parseKerberosParams(kerberos_params.emplace(), config);
    }
    catch (...)
    {
@ -249,7 +295,7 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur
 }

 bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const BasicCredentials & credentials,
-    const LDAPClient::SearchParamsList * search_params, LDAPClient::SearchResultsList * search_results) const
+    const LDAPClient::RoleSearchParamsList * role_search_params, LDAPClient::SearchResultsList * role_search_results) const
 {
    std::optional<LDAPClient::Params> params;
    std::size_t params_hash = 0;
@ -267,9 +313,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
        params->password = credentials.getPassword();

        params->combineCoreHash(params_hash);
-        if (search_params)
+        if (role_search_params)
        {
-            for (const auto & params_instance : *search_params)
+            for (const auto & params_instance : *role_search_params)
            {
                params_instance.combineHash(params_hash);
            }
@ -301,14 +347,14 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B

                        // Ensure that search_params are compatible.
                        (
-                            search_params == nullptr ?
-                            entry.last_successful_search_results.empty() :
-                            search_params->size() == entry.last_successful_search_results.size()
+                            role_search_params == nullptr ?
+                            entry.last_successful_role_search_results.empty() :
+                            role_search_params->size() == entry.last_successful_role_search_results.size()
                        )
                    )
                    {
-                        if (search_results)
-                            *search_results = entry.last_successful_search_results;
+                        if (role_search_results)
+                            *role_search_results = entry.last_successful_role_search_results;

                        return true;
                    }
@ -326,7 +372,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
    }

    LDAPSimpleAuthClient client(params.value());
-    const auto result = client.authenticate(search_params, search_results);
+    const auto result = client.authenticate(role_search_params, role_search_results);
    const auto current_check_timestamp = std::chrono::steady_clock::now();

    // Update the cache, but only if this is the latest check and the server is still configured in a compatible way.
@ -345,9 +391,9 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B

        std::size_t new_params_hash = 0;
        new_params.combineCoreHash(new_params_hash);
-        if (search_params)
+        if (role_search_params)
        {
-            for (const auto & params_instance : *search_params)
+            for (const auto & params_instance : *role_search_params)
            {
                params_instance.combineHash(new_params_hash);
            }
@ -363,17 +409,17 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const B
            entry.last_successful_params_hash = params_hash;
            entry.last_successful_authentication_timestamp = current_check_timestamp;

-            if (search_results)
-                entry.last_successful_search_results = *search_results;
+            if (role_search_results)
+                entry.last_successful_role_search_results = *role_search_results;
            else
-                entry.last_successful_search_results.clear();
+                entry.last_successful_role_search_results.clear();
        }
        else if (
            entry.last_successful_params_hash != params_hash ||
            (
-                search_params == nullptr ?
-                !entry.last_successful_search_results.empty() :
-                search_params->size() != entry.last_successful_search_results.size()
+                role_search_params == nullptr ?
+                !entry.last_successful_role_search_results.empty() :
+                role_search_params->size() != entry.last_successful_role_search_results.size()
            )
        )
        {
--- a/src/Access/ExternalAuthenticators.h
+++ b/src/Access/ExternalAuthenticators.h
@ -34,7 +34,7 @@ public:

    // The name and readiness of the credentials must be verified before calling these.
    bool checkLDAPCredentials(const String & server, const BasicCredentials & credentials,
-        const LDAPClient::SearchParamsList * search_params = nullptr, LDAPClient::SearchResultsList * search_results = nullptr) const;
+        const LDAPClient::RoleSearchParamsList * role_search_params = nullptr, LDAPClient::SearchResultsList * role_search_results = nullptr) const;
    bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const;

    GSSAcceptorContext::Params getKerberosParams() const;
@ -44,7 +44,7 @@ private:
    {
        std::size_t last_successful_params_hash = 0;
        std::chrono::steady_clock::time_point last_successful_authentication_timestamp;
-        LDAPClient::SearchResultsList last_successful_search_results;
+        LDAPClient::SearchResultsList last_successful_role_search_results;
    };

    using LDAPCache = std::unordered_map<String, LDAPCacheEntry>; // user name   -> cache entry
@ -58,4 +58,6 @@ private:
    std::optional<GSSAcceptorContext::Params> kerberos_params;
 };

+void parseLDAPRoleSearchParams(LDAPClient::RoleSearchParams & params, const Poco::Util::AbstractConfiguration & config, const String & prefix);
+
 }
--- a/src/Access/LDAPAccessStorage.cpp
+++ b/src/Access/LDAPAccessStorage.cpp
@ -68,34 +68,15 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m
        common_roles_cfg.insert(role_names.begin(), role_names.end());
    }

-    LDAPClient::SearchParamsList role_search_params_cfg;
+    LDAPClient::RoleSearchParamsList role_search_params_cfg;
    if (has_role_mapping)
    {
        Poco::Util::AbstractConfiguration::Keys all_keys;
        config.keys(prefix, all_keys);
        for (const auto & key : all_keys)
        {
-            if (key != "role_mapping" && key.find("role_mapping[") != 0)
-                continue;
-
-            const String rm_prefix = prefix_str + key;
-            const String rm_prefix_str = rm_prefix + '.';
-            role_search_params_cfg.emplace_back();
-            auto & rm_params = role_search_params_cfg.back();
-
-            rm_params.base_dn = config.getString(rm_prefix_str + "base_dn", "");
-            rm_params.search_filter = config.getString(rm_prefix_str + "search_filter", "");
-            rm_params.attribute = config.getString(rm_prefix_str + "attribute", "cn");
-            rm_params.prefix = config.getString(rm_prefix_str + "prefix", "");
-
-            auto scope = config.getString(rm_prefix_str + "scope", "subtree");
-            boost::algorithm::to_lower(scope);
-            if (scope == "base")           rm_params.scope = LDAPClient::SearchParams::Scope::BASE;
-            else if (scope == "one_level") rm_params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL;
-            else if (scope == "subtree")   rm_params.scope = LDAPClient::SearchParams::Scope::SUBTREE;
-            else if (scope == "children")  rm_params.scope = LDAPClient::SearchParams::Scope::CHILDREN;
-            else
-                throw Exception("Invalid value of 'scope' field in '" + key + "' section of LDAP user directory, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS);
+            if (key == "role_mapping" || key.find("role_mapping[") == 0)
+                parseLDAPRoleSearchParams(role_search_params_cfg.emplace_back(), config, prefix_str + key);
        }
    }

@ -364,7 +345,7 @@ std::set<String> LDAPAccessStorage::mapExternalRolesNoLock(const LDAPClient::Sea


 bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials,
-    const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const
+    const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const
 {
    if (!credentials.isReady())
        return false;
@ -373,7 +354,7 @@ bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const C
        return false;

    if (const auto * basic_credentials = dynamic_cast<const BasicCredentials *>(&credentials))
-        return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &search_results);
+        return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &role_search_results);

    return false;
 }
--- a/src/Access/LDAPAccessStorage.h
+++ b/src/Access/LDAPAccessStorage.h
@ -68,12 +68,12 @@ private:
    void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const;
    std::set<String> mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const;
    bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials,
-        const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const;
+        const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & role_search_results) const;

    mutable std::recursive_mutex mutex;
    AccessControlManager * access_control_manager = nullptr;
    String ldap_server_name;
-    LDAPClient::SearchParamsList role_search_params;
+    LDAPClient::RoleSearchParamsList role_search_params;
    std::set<String> common_role_names;                         // role name that should be granted to all users at all times
    mutable std::map<String, std::size_t> external_role_hashes; // user name -> LDAPClient::SearchResultsList hash (most recently retrieved and processed)
    mutable std::map<String, std::set<String>> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles)
--- a/src/Access/LDAPClient.cpp
+++ b/src/Access/LDAPClient.cpp
@ -32,6 +32,11 @@ void LDAPClient::SearchParams::combineHash(std::size_t & seed) const
    boost::hash_combine(seed, static_cast<int>(scope));
    boost::hash_combine(seed, search_filter);
    boost::hash_combine(seed, attribute);
+}
+
+void LDAPClient::RoleSearchParams::combineHash(std::size_t & seed) const
+{
+    SearchParams::combineHash(seed);
    boost::hash_combine(seed, prefix);
 }

@ -42,6 +47,9 @@ void LDAPClient::Params::combineCoreHash(std::size_t & seed) const
    boost::hash_combine(seed, bind_dn);
    boost::hash_combine(seed, user);
    boost::hash_combine(seed, password);
+
+    if (user_dn_detection)
+        user_dn_detection->combineHash(seed);
 }

 LDAPClient::LDAPClient(const Params & params_)
@ -286,18 +294,33 @@ void LDAPClient::openConnection()
    if (params.enable_tls == LDAPClient::Params::TLSEnable::YES_STARTTLS)
        diag(ldap_start_tls_s(handle, nullptr, nullptr));

+    final_user_name = escapeForLDAP(params.user);
+    final_bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", final_user_name} });
+    final_user_dn = final_bind_dn; // The default value... may be updated right after a successful bind.
+
    switch (params.sasl_mechanism)
    {
        case LDAPClient::Params::SASLMechanism::SIMPLE:
        {
-            const auto escaped_user_name = escapeForLDAP(params.user);
-            const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} });
-
            ::berval cred;
            cred.bv_val = const_cast<char *>(params.password.c_str());
            cred.bv_len = params.password.size();

-            diag(ldap_sasl_bind_s(handle, bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
+            diag(ldap_sasl_bind_s(handle, final_bind_dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr));
+
+            // Once bound, run the user DN search query and update the default value, if asked.
+            if (params.user_dn_detection)
+            {
+                const auto user_dn_search_results = search(*params.user_dn_detection);
+
+                if (user_dn_search_results.empty())
+                    throw Exception("Failed to detect user DN: empty search results", ErrorCodes::LDAP_ERROR);
+
+                if (user_dn_search_results.size() > 1)
+                    throw Exception("Failed to detect user DN: more than one entry in the search results", ErrorCodes::LDAP_ERROR);
+
+                final_user_dn = *user_dn_search_results.begin();
+            }

            break;
        }
@ -316,6 +339,9 @@ void LDAPClient::closeConnection() noexcept

    ldap_unbind_ext_s(handle, nullptr, nullptr);
    handle = nullptr;
+    final_user_name.clear();
+    final_bind_dn.clear();
+    final_user_dn.clear();
 }

 LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
@ -333,10 +359,19 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
        case SearchParams::Scope::CHILDREN:  scope = LDAP_SCOPE_CHILDREN; break;
    }

-    const auto escaped_user_name = escapeForLDAP(params.user);
-    const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} });
-    const auto base_dn = replacePlaceholders(search_params.base_dn, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn} });
-    const auto search_filter = replacePlaceholders(search_params.search_filter, { {"{user_name}", escaped_user_name}, {"{bind_dn}", bind_dn}, {"{base_dn}", base_dn} });
+    const auto final_base_dn = replacePlaceholders(search_params.base_dn, {
+        {"{user_name}", final_user_name},
+        {"{bind_dn}", final_bind_dn},
+        {"{user_dn}", final_user_dn}
+    });
+
+    const auto final_search_filter = replacePlaceholders(search_params.search_filter, {
+        {"{user_name}", final_user_name},
+        {"{bind_dn}", final_bind_dn},
+        {"{user_dn}", final_user_dn},
+        {"{base_dn}", final_base_dn}
+    });
+
    char * attrs[] = { const_cast<char *>(search_params.attribute.c_str()), nullptr };
    ::timeval timeout = { params.search_timeout.count(), 0 };
    LDAPMessage* msgs = nullptr;
@ -349,7 +384,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
        }
    });

-    diag(ldap_search_ext_s(handle, base_dn.c_str(), scope, search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs));
+    diag(ldap_search_ext_s(handle, final_base_dn.c_str(), scope, final_search_filter.c_str(), attrs, 0, nullptr, nullptr, &timeout, params.search_limit, &msgs));

    for (
         auto * msg = ldap_first_message(handle, msgs);
@ -361,6 +396,27 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
        {
            case LDAP_RES_SEARCH_ENTRY:
            {
+                // Extract DN separately, if the requested attribute is DN.
+                if (boost::iequals("dn", search_params.attribute))
+                {
+                    BerElement * ber = nullptr;
+
+                    SCOPE_EXIT({
+                        if (ber)
+                        {
+                            ber_free(ber, 0);
+                            ber = nullptr;
+                        }
+                    });
+
+                    ::berval bv;
+
+                    diag(ldap_get_dn_ber(handle, msg, &ber, &bv));
+
+                    if (bv.bv_val && bv.bv_len > 0)
+                        result.emplace(bv.bv_val, bv.bv_len);
+                }
+
                BerElement * ber = nullptr;

                SCOPE_EXIT({
@ -471,12 +527,12 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params)
    return result;
 }

-bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, SearchResultsList * search_results)
+bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results)
 {
    if (params.user.empty())
        throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS);

-    if (!search_params != !search_results)
+    if (!role_search_params != !role_search_results)
        throw Exception("Cannot return LDAP search results", ErrorCodes::BAD_ARGUMENTS);

    // Silently reject authentication attempt if the password is empty as if it didn't match.
@ -489,21 +545,21 @@ bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params,
    openConnection();

    // While connected, run search queries and save the results, if asked.
-    if (search_params)
+    if (role_search_params)
    {
-        search_results->clear();
-        search_results->reserve(search_params->size());
+        role_search_results->clear();
+        role_search_results->reserve(role_search_params->size());

        try
        {
-            for (const auto & single_search_params : *search_params)
+            for (const auto & params_instance : *role_search_params)
            {
-                search_results->emplace_back(search(single_search_params));
+                role_search_results->emplace_back(search(params_instance));
            }
        }
        catch (...)
        {
-            search_results->clear();
+            role_search_results->clear();
            throw;
        }
    }
@ -532,7 +588,7 @@ LDAPClient::SearchResults LDAPClient::search(const SearchParams &)
    throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
 }

-bool LDAPSimpleAuthClient::authenticate(const SearchParamsList *, SearchResultsList *)
+bool LDAPSimpleAuthClient::authenticate(const RoleSearchParamsList *, SearchResultsList *)
 {
    throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
 }
--- a/src/Access/LDAPClient.h
+++ b/src/Access/LDAPClient.h
@ -38,12 +38,20 @@ public:
        Scope scope = Scope::SUBTREE;
        String search_filter;
        String attribute = "cn";
+
+        void combineHash(std::size_t & seed) const;
+    };
+
+    struct RoleSearchParams
+        : public SearchParams
+    {
        String prefix;

        void combineHash(std::size_t & seed) const;
    };

-    using SearchParamsList = std::vector<SearchParams>;
+    using RoleSearchParamsList = std::vector<RoleSearchParams>;
+
    using SearchResults = std::set<String>;
    using SearchResultsList = std::vector<SearchResults>;

@ -105,6 +113,8 @@ public:
        String user;
        String password;

+        std::optional<SearchParams> user_dn_detection;
+
        std::chrono::seconds verification_cooldown{0};

        std::chrono::seconds operation_timeout{40};
@ -134,6 +144,9 @@ protected:
 #if USE_LDAP
    LDAP * handle = nullptr;
 #endif
+    String final_user_name;
+    String final_bind_dn;
+    String final_user_dn;
 };

 class LDAPSimpleAuthClient
@ -141,7 +154,7 @@ class LDAPSimpleAuthClient
 {
 public:
    using LDAPClient::LDAPClient;
-    bool authenticate(const SearchParamsList * search_params, SearchResultsList * search_results);
+    bool authenticate(const RoleSearchParamsList * role_search_params, SearchResultsList * role_search_results);
 };

 }
--- a/src/Common/tests/gtest_local_address.cpp
+++ b/src/Common/tests/gtest_local_address.cpp
@ -0,0 +1,19 @@
+#include <gtest/gtest.h>
+#include <Common/isLocalAddress.h>
+#include <Common/ShellCommand.h>
+#include <Poco/Net/IPAddress.h>
+#include <IO/ReadHelpers.h>
+
+
+TEST(LocalAddress, SmokeTest)
+{
+    auto cmd = DB::ShellCommand::executeDirect("/bin/hostname", {"-i"});
+    std::string address_str;
+    DB::readString(address_str, cmd->out);
+    cmd->wait();
+    std::cerr << "Got Address:" << address_str << std::endl;
+
+    Poco::Net::IPAddress address(address_str);
+
+    EXPECT_TRUE(DB::isLocalAddress(address));
+}
--- a/src/Compression/tests/gtest_compressionCodec.cpp
+++ b/src/Compression/tests/gtest_compressionCodec.cpp
@ -345,10 +345,12 @@ CodecTestSequence operator*(CodecTestSequence && left, T times)

 std::ostream & operator<<(std::ostream & ostr, const Codec & codec)
 {
-    return ostr << "Codec{"
-                << "name: " << codec.codec_statement
-                << ", expected_compression_ratio: " << *codec.expected_compression_ratio
-                << "}";
+    ostr << "Codec{"
+         << "name: " << codec.codec_statement;
+    if (codec.expected_compression_ratio)
+        return ostr << ", expected_compression_ratio: " << *codec.expected_compression_ratio << "}";
+    else
+        return ostr << "}";
 }

 std::ostream & operator<<(std::ostream & ostr, const CodecTestSequence & seq)
--- a/tests/integration/test_jbod_balancer/test.py
+++ b/tests/integration/test_jbod_balancer/test.py
@ -92,7 +92,10 @@ def test_jbod_balanced_merge(start_cluster):
        node1.query("create table tmp1 as tbl")
        node1.query("create table tmp2 as tbl")

-        for i in range(200):
+        p = Pool(20)
+
+        def task(i):
+            print("Processing insert {}/{}".format(i, 200))
            # around 1k per block
            node1.query(
                "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
@ -104,6 +107,8 @@ def test_jbod_balanced_merge(start_cluster):
                "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
            )

+        p.map(task, range(200))
+
        time.sleep(1)

        check_balance(node1, "tbl")
@ -151,8 +156,10 @@ def test_replicated_balanced_merge_fetch(start_cluster):
            node.query("create table tmp2 as tmp1")

        node2.query("alter table tbl modify setting always_fetch_merged_part = 1")
+        p = Pool(20)

-        for i in range(200):
+        def task(i):
+            print("Processing insert {}/{}".format(i, 200))
            # around 1k per block
            node1.query(
                "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
@ -170,6 +177,8 @@ def test_replicated_balanced_merge_fetch(start_cluster):
                "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)"
            )

+        p.map(task, range(200))
+
        node2.query("SYSTEM SYNC REPLICA tbl", timeout=10)

        check_balance(node1, "tbl")
--- a/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py
+++ b/tests/testflows/ldap/role_mapping/tests/user_dn_detection.py
@ -33,7 +33,7 @@ def check_config(self, entries, valid=True, ldap_server="openldap1", user="user1
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0")
 )
 def config_invalid_base_dn(self):
    """Check when invalid `base_dn` is specified in the user_dn_detection section.
@ -62,7 +62,7 @@ def config_invalid_base_dn(self):
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0")
 )
 def config_empty_base_dn(self):
    """Check when empty `base_dn` is specified in the user_dn_detection section.
@ -90,7 +90,7 @@ def config_empty_base_dn(self):
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0")
 )
 def config_missing_base_dn(self):
    """Check when missing `base_dn` is specified in the user_dn_detection section.
@ -145,7 +145,7 @@ def config_invalid_search_filter(self):
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0")
 )
 def config_missing_search_filter(self):
    """Check when missing `search_filter` is specified in the user_dn_detection section.
@ -172,7 +172,7 @@ def config_missing_search_filter(self):
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0")
 )
 def config_empty_search_filter(self):
    """Check when empty `search_filter` is specified in the user_dn_detection section.
@ -200,7 +200,8 @@ def config_empty_search_filter(self):
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"),
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0")
 )
 def config_valid(self):
    """Check valid config with valid user_dn_detection section.
@ -228,7 +229,8 @@ def config_valid(self):
@TestScenario
@Tags("config")
@Requirements(
-    # FIXME
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_BaseDN("1.0"),
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_SearchFilter("1.0")
 )
 def config_valid_tls_connection(self):
    """Check valid config with valid user_dn_detection section when
@ -256,6 +258,9 @@ def config_valid_tls_connection(self):
    check_config(entries=entries, valid=True, ldap_server="openldap2", user="user2", password="user2")

@TestOutline(Scenario)
+@Requirements(
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection_Scope("1.0")
+)
@Examples("scope base_dn", [
    ("base", "cn=user1,ou=users,dc=company,dc=com"),
    ("one_level","ou=users,dc=company,dc=com"),
@ -399,9 +404,6 @@ def setup_different_bind_dn_and_user_dn(self, uid, map_by, user_dn_detection):
            role_mappings=role_mappings, restart=True)

@TestScenario
-@Requirements(
-    # FIXME:
-)
 def map_roles_by_user_dn_when_base_dn_and_user_dn_are_different(self):
    """Check the case when we map roles using user_dn then
    the first user has uid of second user and second user
@ -429,9 +431,6 @@ def map_roles_by_user_dn_when_base_dn_and_user_dn_are_different(self):
        assert f"GRANT role0_{uid} TO second_user" in r.output, error()

@TestScenario
-@Requirements(
-    # FIXME:
-)
 def map_roles_by_bind_dn_when_base_dn_and_user_dn_are_different(self):
    """Check the case when we map roles by bind_dn when bind_dn and user_dn
    are different.
@ -457,7 +456,7 @@ def map_roles_by_bind_dn_when_base_dn_and_user_dn_are_different(self):
@TestFeature
@Name("user dn detection")
@Requirements(
-    #RQ_SRS_014_LDAP_UserDNDetection("1.0")
+    RQ_SRS_014_LDAP_RoleMapping_Configuration_Server_UserDNDetection("1.0")
 )
 def feature(self):
    """Check LDAP user DN detection.
--- a/tests/testflows/regression.py
+++ b/tests/testflows/regression.py
@ -29,7 +29,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None):
            run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args)
            run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args)
            run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args)
-            # run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args)
+            #run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args)
        finally:
            join(tasks)