Implement LDAPAccessStorage and integrate it into AccessControlManager

Rename ExternalAuthenticators::setConfig to setConfiguration
Revisit LDAP servers config section comments
Add user_directories config section with comments (only for ldap)
Fix bug in MemoryAccessStorage::insertImpl

programs/server/Server.cpp

@@ -595,6 +595,14 @@ int Server::main(const std::vector<std::string> & /*args*/)
     if (!access_control_local_path.empty())
         global_context->getAccessControlManager().setLocalDirectory(access_control_local_path);
 
+    /// Set LDAP user directory config.
+    const bool has_ldap_directory_config = config().has("user_directories.ldap");
+    if (has_ldap_directory_config) {
+        auto ldap_directory_config = config().createView("user_directories.ldap");
+        if (ldap_directory_config)
+            global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config);
+    }
+
     /// Limit on total number of concurrently executed queries.
     global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0));
 

programs/server/config.xml

@@ -215,10 +215,10 @@
     <!-- Path to folder where users and roles created by SQL commands are stored. -->
     <access_control_path>/var/lib/clickhouse/access/</access_control_path>
 
-    <!-- External user directories (LDAP). -->
+    <!-- LDAP server definitions. -->
     <ldap_servers>
         <!-- List LDAP servers with their connection parameters here to later use them as authenticators for dedicated users,
-             who have 'ldap' authentication mechanism specified instead of 'password'.
+              who have 'ldap' authentication mechanism specified instead of 'password'.
              Parameters:
                 host - LDAP server hostname or IP, this parameter is mandatory and cannot be empty.
                 port - LDAP server port, default is 636 if enable_tls is set to true, 389 otherwise.
@@ -237,7 +237,7 @@
                 tls_key_file - path to certificate key file.
                 tls_ca_cert_file - path to CA certificate file.
                 tls_ca_cert_dir - path to the directory containing CA certificates.
-                tls_cipher_suite - allowed cipher suite.
+                tls_cipher_suite - allowed cipher suite (in OpenSSL notation).
              Example:
                 <my_ldap_server>
                     <host>localhost</host>
@@ -256,6 +256,23 @@
         -->
     </ldap_servers>
 
+    <!-- User directories (LDAP, etc.) -->
+    <user_directories>
+        <!-- To add an LDAP server as a user directory, define a single 'ldap' section with the following parameters:
+                server - one of LDAP server names defined in 'ldap_servers' config section above.
+                        This parameter is mandatory and cannot be empty.
+                user_template - name of a user that will be used as a template for all users retrieved from the LDAP server.
+                        Each LDAP user will inherit all parameters from 'user_template' user, except for the authentication method,
+                         which will be set to 'ldap' and the 'server' defined above will be used as a LDAP server for authentication.
+                        This parameter is optional, if not specified, 'default' user will be used as a template.
+             Example:
+                <ldap>
+                    <server>my_ldap_server</server>
+                    <user_template>my_user</user_template>
+                </ldap>
+        -->
+    </user_directories>
+
     <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
     <users_config>users.xml</users_config>
 

src/Access/AccessControlManager.cpp

@@ -3,6 +3,7 @@
 #include <Access/MemoryAccessStorage.h>
 #include <Access/UsersConfigAccessStorage.h>
 #include <Access/DiskAccessStorage.h>
+#include <Access/LDAPAccessStorage.h>
 #include <Access/ContextAccess.h>
 #include <Access/RoleCache.h>
 #include <Access/RowPolicyCache.h>
@@ -28,11 +29,14 @@ namespace
 #if 0  /// Memory access storage is disabled.
         list.emplace_back(std::make_unique<MemoryAccessStorage>());
 #endif
+
+        list.emplace_back(std::make_unique<LDAPAccessStorage>());
         return list;
     }
 
     constexpr size_t DISK_ACCESS_STORAGE_INDEX = 0;
     constexpr size_t USERS_CONFIG_ACCESS_STORAGE_INDEX = 1;
+    constexpr size_t LDAP_ACCESS_STORAGE_INDEX = 2;
 }
 
 
@@ -81,12 +85,6 @@ void AccessControlManager::setLocalDirectory(const String & directory_path)
 }
 
 
-void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config)
-{
-    external_authenticators->setConfig(config, getLogger());
-}
-
-
 void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguration & users_config)
 {
     auto & users_config_access_storage = dynamic_cast<UsersConfigAccessStorage &>(getStorageByIndex(USERS_CONFIG_ACCESS_STORAGE_INDEX));
@@ -94,6 +92,19 @@ void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguratio
 }
 
 
+void AccessControlManager::setLDAPConfig(const Poco::Util::AbstractConfiguration & users_config)
+{
+    auto & ldap_access_storage = dynamic_cast<LDAPAccessStorage &>(getStorageByIndex(LDAP_ACCESS_STORAGE_INDEX));
+    ldap_access_storage.setConfiguration(users_config, this);
+}
+
+
+void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config)
+{
+    external_authenticators->setConfiguration(config, getLogger());
+}
+
+
 void AccessControlManager::setDefaultProfileName(const String & default_profile_name)
 {
     settings_profiles_cache->setDefaultProfileName(default_profile_name);

src/Access/AccessControlManager.h

@@ -49,8 +49,10 @@ public:
     ~AccessControlManager();
 
     void setLocalDirectory(const String & directory);
-    void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
     void setUsersConfig(const Poco::Util::AbstractConfiguration & users_config);
+    void setLDAPConfig(const Poco::Util::AbstractConfiguration & users_config);
+
+    void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
     void setDefaultProfileName(const String & default_profile_name);
 
     std::shared_ptr<const ContextAccess> getContextAccess(

src/Access/ExternalAuthenticators.cpp

@@ -156,7 +156,7 @@ void ExternalAuthenticators::reset()
     ldap_server_params.clear();
 }
 
-void ExternalAuthenticators::setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
+void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
 {
     std::scoped_lock lock(mutex);
     reset();

src/Access/ExternalAuthenticators.h

@@ -26,7 +26,7 @@ class ExternalAuthenticators
 {
 public:
     void reset();
-    void setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log);
+    void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log);
 
     void setLDAPServerParams(const String & server, const LDAPServerParams & params);
     LDAPServerParams getLDAPServerParams(const String & server) const;

src/Access/LDAPAccessStorage.cpp

@@ -0,0 +1,175 @@
+#include <Access/LDAPAccessStorage.h>
+#include <Access/User.h>
+#include <common/logger_useful.h>
+#include <ext/scope_guard.h>
+#include <Poco/Util/AbstractConfiguration.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+    extern const int UNKNOWN_ADDRESS_PATTERN_TYPE;
+    extern const int NOT_IMPLEMENTED;
+}
+
+
+LDAPAccessStorage::LDAPAccessStorage() : IAccessStorage("ldap")
+{
+}
+
+
+void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration & config, IAccessStorage * top_enclosing_storage_)
+{
+    std::scoped_lock lock(mutex);
+
+    const bool has_server = config.has("server");
+    const bool has_user_template = config.has("user_template");
+
+    if (!has_server)
+        throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
+
+    const auto ldap_server_ = config.getString("server");
+    const auto user_template_ = (has_user_template ? config.getString("user_template") : "default");
+
+    if (ldap_server_.empty())
+        throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
+
+    if (user_template_.empty())
+        throw Exception("Empty 'user_template' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS);
+
+    ldap_server = ldap_server_;
+    user_template = user_template_;
+    top_enclosing_storage = top_enclosing_storage_;
+}
+
+
+bool LDAPAccessStorage::isConfiguredNoLock() const
+{
+    return !ldap_server.empty() && !user_template.empty() && top_enclosing_storage;
+}
+
+
+std::optional<UUID> LDAPAccessStorage::findImpl(EntityType type, const String & name) const
+{
+    if (type == EntityType::USER)
+    {
+        std::scoped_lock lock(mutex);
+
+        // Detect and avoid loops/duplicate creations.
+        if (helper_lookup_in_progress)
+            return {};
+
+        helper_lookup_in_progress = true;
+        SCOPE_EXIT({ helper_lookup_in_progress = false; });
+
+        // Return the id immediately if we already have it.
+        const auto id = memory_storage.find(type, name);
+        if (id.has_value())
+            return id;
+
+        if (!isConfiguredNoLock())
+        {
+            LOG_WARNING(getLogger(), "Access storage instance is not configured.");
+            return {};
+        }
+
+        // Stop if entity exists anywhere else, to avoid duplicates.
+        if (top_enclosing_storage->find(type, name))
+            return {};
+
+        // Entity doesn't exist. We are going to create it. Here we retrieve the template first.
+        const auto user_tmp = top_enclosing_storage->read<User>(user_template);
+        if (!user_tmp)
+        {
+            LOG_WARNING(getLogger(), "Unable to retrieve user template '{}': user does not exist in access storage '{}'.", user_template, top_enclosing_storage->getStorageName());
+            return {};
+        }
+
+        // Build the new entity based on the existing template.
+        const auto user = std::make_shared<User>(*user_tmp);
+        user->setName(name);
+        user->authentication = Authentication(Authentication::Type::LDAP_SERVER);
+        user->authentication.setServerName(ldap_server);
+
+        return memory_storage.insert(user);
+    }
+
+    return memory_storage.find(type, name);
+}
+
+
+std::vector<UUID> LDAPAccessStorage::findAllImpl(EntityType type) const
+{
+    return memory_storage.findAll(type);
+}
+
+
+bool LDAPAccessStorage::existsImpl(const UUID & id) const
+{
+    return memory_storage.exists(id);
+}
+
+
+AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const
+{
+    return memory_storage.read(id);
+}
+
+
+String LDAPAccessStorage::readNameImpl(const UUID & id) const
+{
+    return memory_storage.readName(id);
+}
+
+
+bool LDAPAccessStorage::canInsertImpl(const AccessEntityPtr &) const
+{
+    return false;
+}
+
+
+UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool)
+{
+    throwReadonlyCannotInsert(entity->getType(), entity->getName());
+}
+
+
+void LDAPAccessStorage::removeImpl(const UUID & id)
+{
+    auto entity = read(id);
+    throwReadonlyCannotRemove(entity->getType(), entity->getName());
+}
+
+
+void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &)
+{
+    auto entity = read(id);
+    throwReadonlyCannotUpdate(entity->getType(), entity->getName());
+}
+
+
+ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const
+{
+    return memory_storage.subscribeForChanges(id, handler);
+}
+
+
+ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const
+{
+    return memory_storage.subscribeForChanges(type, handler);
+}
+
+
+bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const
+{
+    return memory_storage.hasSubscription(id);
+}
+
+
+bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const
+{
+    return memory_storage.hasSubscription(type);
+}
+}

src/Access/LDAPAccessStorage.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <Access/MemoryAccessStorage.h>
+#include <Core/Types.h>
+#include <mutex>
+
+
+namespace Poco
+{
+    namespace Util
+    {
+        class AbstractConfiguration;
+    }
+}
+
+
+namespace DB
+{
+/// Implementation of IAccessStorage which allows attaching users from a remote LDAP server.
+/// Currently, any user name will be treated as a name of an existing remote user,
+/// a user info entity will be created, with LDAP_SERVER authentication type.
+class LDAPAccessStorage : public IAccessStorage
+{
+public:
+    LDAPAccessStorage();
+
+    void setConfiguration(const Poco::Util::AbstractConfiguration & config, IAccessStorage * top_enclosing_storage_);
+
+private: // IAccessStorage implementations.
+    std::optional<UUID> findImpl(EntityType type, const String & name) const override;
+    std::vector<UUID> findAllImpl(EntityType type) const override;
+    bool existsImpl(const UUID & id) const override;
+    AccessEntityPtr readImpl(const UUID & id) const override;
+    String readNameImpl(const UUID & id) const override;
+    bool canInsertImpl(const AccessEntityPtr &) const override;
+    UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
+    void removeImpl(const UUID & id) override;
+    void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
+    ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
+    ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override;
+    bool hasSubscriptionImpl(const UUID & id) const override;
+    bool hasSubscriptionImpl(EntityType type) const override;
+
+private:
+    bool isConfiguredNoLock() const;
+
+    mutable std::recursive_mutex mutex;
+    String ldap_server;
+    String user_template;
+    IAccessStorage * top_enclosing_storage = nullptr;
+    mutable bool helper_lookup_in_progress = false;
+    mutable MemoryAccessStorage memory_storage;
+};
+}

src/Access/MemoryAccessStorage.cpp

@@ -69,7 +69,7 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re
 
     UUID id = generateRandomID();
     std::lock_guard lock{mutex};
-    insertNoLock(generateRandomID(), new_entity, replace_if_exists, notifications);
+    insertNoLock(id, new_entity, replace_if_exists, notifications);
     return id;
 }
 

src/Access/ya.make

@@ -21,6 +21,7 @@ SRCS(
     GrantedRoles.cpp
     IAccessEntity.cpp
     IAccessStorage.cpp
+    LDAPAccessStorage.cpp
     LDAPClient.cpp
     MemoryAccessStorage.cpp
     MultipleAccessStorage.cpp