From 5d6b5101fe2049ce2704e0c022d45a487a84ccec Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 23 Jul 2020 21:55:24 +0400 Subject: [PATCH 001/441] Implement LDAPAccessStorage and integrate it into AccessControlManager Rename ExternalAuthenticators::setConfig to setConfiguration Revisit LDAP servers config section comments Add user_directories config section with comments (only for ldap) Fix bug in MemoryAccessStorage::insertImpl --- programs/server/Server.cpp | 8 ++ programs/server/config.xml | 23 +++- src/Access/AccessControlManager.cpp | 23 +++- src/Access/AccessControlManager.h | 4 +- src/Access/ExternalAuthenticators.cpp | 2 +- src/Access/ExternalAuthenticators.h | 2 +- src/Access/LDAPAccessStorage.cpp | 175 ++++++++++++++++++++++++++ src/Access/LDAPAccessStorage.h | 54 ++++++++ src/Access/MemoryAccessStorage.cpp | 2 +- src/Access/ya.make | 1 + 10 files changed, 281 insertions(+), 13 deletions(-) create mode 100644 src/Access/LDAPAccessStorage.cpp create mode 100644 src/Access/LDAPAccessStorage.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c3b17824151..eb03deec072 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -595,6 +595,14 @@ int Server::main(const std::vector & /*args*/) if (!access_control_local_path.empty()) global_context->getAccessControlManager().setLocalDirectory(access_control_local_path); + /// Set LDAP user directory config. + const bool has_ldap_directory_config = config().has("user_directories.ldap"); + if (has_ldap_directory_config) { + auto ldap_directory_config = config().createView("user_directories.ldap"); + if (ldap_directory_config) + global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config); + } + /// Limit on total number of concurrently executed queries. global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0)); diff --git a/programs/server/config.xml b/programs/server/config.xml index 3e01964f0ff..b09a53ccf98 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -215,10 +215,10 @@ /var/lib/clickhouse/access/ - + + + + + + users.xml diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 5966c1aff75..3f457009a5b 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -28,11 +29,14 @@ namespace #if 0 /// Memory access storage is disabled. list.emplace_back(std::make_unique()); #endif + + list.emplace_back(std::make_unique()); return list; } constexpr size_t DISK_ACCESS_STORAGE_INDEX = 0; constexpr size_t USERS_CONFIG_ACCESS_STORAGE_INDEX = 1; + constexpr size_t LDAP_ACCESS_STORAGE_INDEX = 2; } @@ -81,12 +85,6 @@ void AccessControlManager::setLocalDirectory(const String & directory_path) } -void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) -{ - external_authenticators->setConfig(config, getLogger()); -} - - void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguration & users_config) { auto & users_config_access_storage = dynamic_cast(getStorageByIndex(USERS_CONFIG_ACCESS_STORAGE_INDEX)); @@ -94,6 +92,19 @@ void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguratio } +void AccessControlManager::setLDAPConfig(const Poco::Util::AbstractConfiguration & users_config) +{ + auto & ldap_access_storage = dynamic_cast(getStorageByIndex(LDAP_ACCESS_STORAGE_INDEX)); + ldap_access_storage.setConfiguration(users_config, this); +} + + +void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) +{ + external_authenticators->setConfiguration(config, getLogger()); +} + + void AccessControlManager::setDefaultProfileName(const String & default_profile_name) { settings_profiles_cache->setDefaultProfileName(default_profile_name); diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 467b7471423..08aa447275b 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -49,8 +49,10 @@ public: ~AccessControlManager(); void setLocalDirectory(const String & directory); - void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); void setUsersConfig(const Poco::Util::AbstractConfiguration & users_config); + void setLDAPConfig(const Poco::Util::AbstractConfiguration & users_config); + + void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); void setDefaultProfileName(const String & default_profile_name); std::shared_ptr getContextAccess( diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index a0c5fbf1a79..3ed1b21c3c2 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -156,7 +156,7 @@ void ExternalAuthenticators::reset() ldap_server_params.clear(); } -void ExternalAuthenticators::setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) +void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) { std::scoped_lock lock(mutex); reset(); diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 54af87604a6..5dde82b68df 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -26,7 +26,7 @@ class ExternalAuthenticators { public: void reset(); - void setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log); + void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log); void setLDAPServerParams(const String & server, const LDAPServerParams & params); LDAPServerParams getLDAPServerParams(const String & server) const; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp new file mode 100644 index 00000000000..314beca4dcc --- /dev/null +++ b/src/Access/LDAPAccessStorage.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_ADDRESS_PATTERN_TYPE; + extern const int NOT_IMPLEMENTED; +} + + +LDAPAccessStorage::LDAPAccessStorage() : IAccessStorage("ldap") +{ +} + + +void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration & config, IAccessStorage * top_enclosing_storage_) +{ + std::scoped_lock lock(mutex); + + const bool has_server = config.has("server"); + const bool has_user_template = config.has("user_template"); + + if (!has_server) + throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); + + const auto ldap_server_ = config.getString("server"); + const auto user_template_ = (has_user_template ? config.getString("user_template") : "default"); + + if (ldap_server_.empty()) + throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); + + if (user_template_.empty()) + throw Exception("Empty 'user_template' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); + + ldap_server = ldap_server_; + user_template = user_template_; + top_enclosing_storage = top_enclosing_storage_; +} + + +bool LDAPAccessStorage::isConfiguredNoLock() const +{ + return !ldap_server.empty() && !user_template.empty() && top_enclosing_storage; +} + + +std::optional LDAPAccessStorage::findImpl(EntityType type, const String & name) const +{ + if (type == EntityType::USER) + { + std::scoped_lock lock(mutex); + + // Detect and avoid loops/duplicate creations. + if (helper_lookup_in_progress) + return {}; + + helper_lookup_in_progress = true; + SCOPE_EXIT({ helper_lookup_in_progress = false; }); + + // Return the id immediately if we already have it. + const auto id = memory_storage.find(type, name); + if (id.has_value()) + return id; + + if (!isConfiguredNoLock()) + { + LOG_WARNING(getLogger(), "Access storage instance is not configured."); + return {}; + } + + // Stop if entity exists anywhere else, to avoid duplicates. + if (top_enclosing_storage->find(type, name)) + return {}; + + // Entity doesn't exist. We are going to create it. Here we retrieve the template first. + const auto user_tmp = top_enclosing_storage->read(user_template); + if (!user_tmp) + { + LOG_WARNING(getLogger(), "Unable to retrieve user template '{}': user does not exist in access storage '{}'.", user_template, top_enclosing_storage->getStorageName()); + return {}; + } + + // Build the new entity based on the existing template. + const auto user = std::make_shared(*user_tmp); + user->setName(name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + return memory_storage.insert(user); + } + + return memory_storage.find(type, name); +} + + +std::vector LDAPAccessStorage::findAllImpl(EntityType type) const +{ + return memory_storage.findAll(type); +} + + +bool LDAPAccessStorage::existsImpl(const UUID & id) const +{ + return memory_storage.exists(id); +} + + +AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const +{ + return memory_storage.read(id); +} + + +String LDAPAccessStorage::readNameImpl(const UUID & id) const +{ + return memory_storage.readName(id); +} + + +bool LDAPAccessStorage::canInsertImpl(const AccessEntityPtr &) const +{ + return false; +} + + +UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) +{ + throwReadonlyCannotInsert(entity->getType(), entity->getName()); +} + + +void LDAPAccessStorage::removeImpl(const UUID & id) +{ + auto entity = read(id); + throwReadonlyCannotRemove(entity->getType(), entity->getName()); +} + + +void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) +{ + auto entity = read(id); + throwReadonlyCannotUpdate(entity->getType(), entity->getName()); +} + + +ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const +{ + return memory_storage.subscribeForChanges(id, handler); +} + + +ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const +{ + return memory_storage.subscribeForChanges(type, handler); +} + + +bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const +{ + return memory_storage.hasSubscription(id); +} + + +bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const +{ + return memory_storage.hasSubscription(type); +} +} diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h new file mode 100644 index 00000000000..2978724aff7 --- /dev/null +++ b/src/Access/LDAPAccessStorage.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + + +namespace Poco +{ + namespace Util + { + class AbstractConfiguration; + } +} + + +namespace DB +{ +/// Implementation of IAccessStorage which allows attaching users from a remote LDAP server. +/// Currently, any user name will be treated as a name of an existing remote user, +/// a user info entity will be created, with LDAP_SERVER authentication type. +class LDAPAccessStorage : public IAccessStorage +{ +public: + LDAPAccessStorage(); + + void setConfiguration(const Poco::Util::AbstractConfiguration & config, IAccessStorage * top_enclosing_storage_); + +private: // IAccessStorage implementations. + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; + bool existsImpl(const UUID & id) const override; + AccessEntityPtr readImpl(const UUID & id) const override; + String readNameImpl(const UUID & id) const override; + bool canInsertImpl(const AccessEntityPtr &) const override; + UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override; + void removeImpl(const UUID & id) override; + void updateImpl(const UUID & id, const UpdateFunc & update_func) override; + ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; + bool hasSubscriptionImpl(const UUID & id) const override; + bool hasSubscriptionImpl(EntityType type) const override; + +private: + bool isConfiguredNoLock() const; + + mutable std::recursive_mutex mutex; + String ldap_server; + String user_template; + IAccessStorage * top_enclosing_storage = nullptr; + mutable bool helper_lookup_in_progress = false; + mutable MemoryAccessStorage memory_storage; +}; +} diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 720b82796b7..20bea4e472c 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -69,7 +69,7 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re UUID id = generateRandomID(); std::lock_guard lock{mutex}; - insertNoLock(generateRandomID(), new_entity, replace_if_exists, notifications); + insertNoLock(id, new_entity, replace_if_exists, notifications); return id; } diff --git a/src/Access/ya.make b/src/Access/ya.make index 175bb86d737..b74642fb95b 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -21,6 +21,7 @@ SRCS( GrantedRoles.cpp IAccessEntity.cpp IAccessStorage.cpp + LDAPAccessStorage.cpp LDAPClient.cpp MemoryAccessStorage.cpp MultipleAccessStorage.cpp From 3b3404c326bd80314f2e3e1525903f08f6ee033e Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 23 Jul 2020 22:10:57 +0400 Subject: [PATCH 002/441] Style fix Remove unused declarations --- programs/server/Server.cpp | 3 ++- src/Access/LDAPAccessStorage.cpp | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index eb03deec072..bd04dc963be 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -597,7 +597,8 @@ int Server::main(const std::vector & /*args*/) /// Set LDAP user directory config. const bool has_ldap_directory_config = config().has("user_directories.ldap"); - if (has_ldap_directory_config) { + if (has_ldap_directory_config) + { auto ldap_directory_config = config().createView("user_directories.ldap"); if (ldap_directory_config) global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config); diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 314beca4dcc..ab705ab303c 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -10,8 +10,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int UNKNOWN_ADDRESS_PATTERN_TYPE; - extern const int NOT_IMPLEMENTED; } From 79332c561e6c9b7d2a499d5de9f73749778393da Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 13:52:03 +0400 Subject: [PATCH 003/441] Fix local variable naming --- src/Access/LDAPAccessStorage.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index ab705ab303c..b67459aa322 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -28,17 +28,17 @@ void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration if (!has_server) throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - const auto ldap_server_ = config.getString("server"); - const auto user_template_ = (has_user_template ? config.getString("user_template") : "default"); + const auto ldap_server_cfg = config.getString("server"); + const auto user_template_cfg = (has_user_template ? config.getString("user_template") : "default"); - if (ldap_server_.empty()) + if (ldap_server_cfg.empty()) throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - if (user_template_.empty()) + if (user_template_cfg.empty()) throw Exception("Empty 'user_template' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - ldap_server = ldap_server_; - user_template = user_template_; + ldap_server = ldap_server_cfg; + user_template = user_template_cfg; top_enclosing_storage = top_enclosing_storage_; } From 6c1643d3c54e903d9e5c400ccfbc4dbf1b87bba1 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 14:11:00 +0400 Subject: [PATCH 004/441] Remove unneeded logging --- src/Access/LDAPAccessStorage.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index b67459aa322..16d985064a7 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -68,16 +68,15 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & return id; if (!isConfiguredNoLock()) - { - LOG_WARNING(getLogger(), "Access storage instance is not configured."); return {}; - } // Stop if entity exists anywhere else, to avoid duplicates. if (top_enclosing_storage->find(type, name)) return {}; - // Entity doesn't exist. We are going to create it. Here we retrieve the template first. + // Entity doesn't exist. We are going to create one. + + // Retrieve the template first. const auto user_tmp = top_enclosing_storage->read(user_template); if (!user_tmp) { From bda2eeef17bbf4c1750484b133bf562dccf05efb Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 19:38:33 +0400 Subject: [PATCH 005/441] Fixing timeouts, courtesy of @vzakaznikov --- tests/testflows/helpers/cluster.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 9f86d44124c..6d3ae97e000 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -167,17 +167,20 @@ class Cluster(object): self.docker_compose += f" --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" self.lock = threading.Lock() - def shell(self, node): + def shell(self, node, timeout=120): """Returns unique shell terminal to be used. """ if node is None: return Shell() - return Shell(command=[ + shell = Shell(command=[ "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" ], name=node) - def bash(self, node, timeout=60): + shell.timeout = timeout + return shell + + def bash(self, node, timeout=120): """Returns thread-local bash terminal to a specific node. From 90a064c7a62a0c6ddb0a114a11e5f8803d898398 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 19:39:18 +0400 Subject: [PATCH 006/441] Fix compilation --- programs/server/Server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bd04dc963be..d579ea40316 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -599,7 +599,7 @@ int Server::main(const std::vector & /*args*/) const bool has_ldap_directory_config = config().has("user_directories.ldap"); if (has_ldap_directory_config) { - auto ldap_directory_config = config().createView("user_directories.ldap"); + auto * ldap_directory_config = config().createView("user_directories.ldap"); if (ldap_directory_config) global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config); } From 479fa4c3254eb445c85092de4df238453f4abe01 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 27 Jul 2020 14:24:56 +0400 Subject: [PATCH 007/441] Improve LDAP-related comments --- programs/server/config.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index b09a53ccf98..d82e3b6a72c 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -217,8 +217,8 @@ - - diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 06347412da7..9dcb1cdd4f9 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -157,7 +157,6 @@ void AccessControlManager::addUsersConfigStorage(const Poco::Util::AbstractConfi addUsersConfigStorage(UsersConfigAccessStorage::STORAGE_TYPE, users_config_); } - void AccessControlManager::addUsersConfigStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & users_config_) { auto check_setting_name_function = [this](const std::string_view & setting_name) { checkSettingNameIsAllowed(setting_name); }; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index a98eab5a2b0..8285239abb0 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -1,5 +1,7 @@ #include +#include #include +#include #include #include #include @@ -20,39 +22,42 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_) } -void LDAPAccessStorage::setConfiguration(IAccessStorage * top_enclosing_storage_, const Poco::Util::AbstractConfiguration & config, const String & prefix) +void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix) { + // TODO: switch to passing config as a ConfigurationView and remove this extra prefix once a version of Poco with proper implementation is available. const String prefix_str = (prefix.empty() ? "" : prefix + "."); std::scoped_lock lock(mutex); const bool has_server = config.has(prefix_str + "server"); - const bool has_user_template = config.has(prefix_str + "user_template"); + const bool has_roles = config.has(prefix_str + "roles"); if (!has_server) throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); const auto ldap_server_cfg = config.getString(prefix_str + "server"); - String user_template_cfg; - if (ldap_server_cfg.empty()) throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - if (has_user_template) - user_template_cfg = config.getString(prefix_str + "user_template"); + std::set roles_cfg; + if (has_roles) + { + Poco::Util::AbstractConfiguration::Keys role_names; + config.keys(prefix_str + "roles", role_names); - if (user_template_cfg.empty()) - user_template_cfg = "default"; + // Currently, we only extract names of roles from the section names and assign them directly and unconditionally. + roles_cfg.insert(role_names.begin(), role_names.end()); + } ldap_server = ldap_server_cfg; - user_template = user_template_cfg; - top_enclosing_storage = top_enclosing_storage_; + roles.swap(roles_cfg); + access_control_manager = access_control_manager_; } bool LDAPAccessStorage::isConfiguredNoLock() const { - return !ldap_server.empty() && !user_template.empty() && top_enclosing_storage; + return !ldap_server.empty() &&/* !roles.empty() &&*/ access_control_manager; } @@ -74,13 +79,6 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & { std::scoped_lock lock(mutex); - // Detect and avoid loops/duplicate creations. - if (helper_lookup_in_progress) - return {}; - - helper_lookup_in_progress = true; - SCOPE_EXIT({ helper_lookup_in_progress = false; }); - // Return the id immediately if we already have it. const auto id = memory_storage.find(type, name); if (id.has_value()) @@ -89,32 +87,39 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & if (!isConfiguredNoLock()) return {}; - // Stop if entity exists anywhere else, to avoid duplicates. - if (top_enclosing_storage->find(type, name)) - return {}; + // Stop if entity exists anywhere else, to avoid generating duplicates. + auto * this_base = dynamic_cast(this); + const auto storages = access_control_manager->getStoragesPtr(); + for (const auto & storage : *storages) + { + if (storage.get() != this_base && storage->find(type, name)) + return {}; + } // Entity doesn't exist. We are going to create one. - - // Retrieve the template first. - std::shared_ptr user_tmp; - try - { - user_tmp = top_enclosing_storage->read(user_template); - if (!user_tmp) - throw Exception("Retrieved user is empty", IAccessEntity::TypeInfo::get(IAccessEntity::Type::USER).not_found_error_code); - } - catch (...) - { - tryLogCurrentException(getLogger(), "Unable to retrieve user template '" + user_template + "' from access storage '" + top_enclosing_storage->getStorageName() + "'"); - return {}; - } - - // Build the new entity based on the existing template. - const auto user = std::make_shared(*user_tmp); + const auto user = std::make_shared(); user->setName(name); user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); + for (const auto& role_name : roles) { + std::optional role_id; + + try + { + role_id = access_control_manager->find(role_name); + if (!role_id) + throw Exception("Retrieved role info is empty", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); + } + catch (...) + { + tryLogCurrentException(getLogger(), "Unable to retrieve role '" + role_name + "' info from access storage '" + access_control_manager->getStorageName() + "'"); + return {}; + } + + user->granted_roles.grant(role_id.value()); + } + return memory_storage.insert(user); } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 90a87eca292..eb8b5cac1bc 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace Poco @@ -16,6 +17,8 @@ namespace Poco namespace DB { +class AccessControlManager; + /// Implementation of IAccessStorage which allows attaching users from a remote LDAP server. /// Currently, any user name will be treated as a name of an existing remote user, /// a user info entity will be created, with LDAP_SERVER authentication type. @@ -27,7 +30,7 @@ public: explicit LDAPAccessStorage(const String & storage_name_ = STORAGE_TYPE); virtual ~LDAPAccessStorage() override = default; - void setConfiguration(IAccessStorage * top_enclosing_storage_, const Poco::Util::AbstractConfiguration & config, const String & prefix = ""); + void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix = ""); public: // IAccessStorage implementations. virtual const char * getStorageType() const override; @@ -53,9 +56,8 @@ private: mutable std::recursive_mutex mutex; String ldap_server; - String user_template; - IAccessStorage * top_enclosing_storage = nullptr; - mutable bool helper_lookup_in_progress = false; + std::set roles; + AccessControlManager * access_control_manager = nullptr; mutable MemoryAccessStorage memory_storage; }; } From ec52a165af469c0d71fdc9d54773060b74cad21c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 20 Aug 2020 12:46:42 +0400 Subject: [PATCH 012/441] Style fixes --- programs/server/config.xml | 2 +- src/Access/LDAPAccessStorage.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index cdae1c5409e..26ca26087ba 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -262,7 +262,7 @@ with the following parameters: server - one of LDAP server names defined in 'ldap_servers' config section above. This parameter is mandatory and cannot be empty. - roles - section with a list of locally defined roles that will be granted to each user retrieved from the LDAP server. + roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. If any of the listed roles is not defined locally at the time of authentication, the authenthication attept will fail as if the provided password was incorrect. diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 8285239abb0..92aef53af64 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -88,7 +88,7 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & return {}; // Stop if entity exists anywhere else, to avoid generating duplicates. - auto * this_base = dynamic_cast(this); + const auto * this_base = dynamic_cast(this); const auto storages = access_control_manager->getStoragesPtr(); for (const auto & storage : *storages) { @@ -102,7 +102,8 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); - for (const auto& role_name : roles) { + for (const auto& role_name : roles) + { std::optional role_id; try From 1cc9b81cddd3d0ba2cc0a205678e9f858b829652 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 24 Aug 2020 16:02:47 +0400 Subject: [PATCH 013/441] Disallow multiple ldap sections in user_directories section --- src/Access/AccessControlManager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 9dcb1cdd4f9..c1a6baf4972 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -24,6 +24,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int UNKNOWN_SETTING; } @@ -250,7 +251,8 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( String prefix = key + "." + key_in_user_directories; String type = key_in_user_directories; - if (size_t bracket_pos = type.find('['); bracket_pos != String::npos) + const size_t bracket_pos = type.find('['); + if (bracket_pos != String::npos) type.resize(bracket_pos); if ((type == "users_xml") || (type == "users_config")) type = UsersConfigAccessStorage::STORAGE_TYPE; @@ -280,6 +282,8 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( } else if (type == LDAPAccessStorage::STORAGE_TYPE) { + if (bracket_pos != String::npos) + throw Exception("Duplicate storage type '" + type + "' at " + prefix + " in config", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); addLDAPStorage(name, config, prefix); } else From 1768db2c2f1c3f3342526f38399c058500823144 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Wed, 26 Aug 2020 22:09:26 +0400 Subject: [PATCH 014/441] Add findOrGenerate() to IAccessStorage interface (falls back to find() by default) Use findOrGenerate() when setting user in Context (authentication path) Implement findOrGenerateImpl() and findImpl() in LDAPAccessStorage --- src/Access/IAccessStorage.cpp | 12 ++++++++++++ src/Access/IAccessStorage.h | 7 +++++++ src/Access/LDAPAccessStorage.cpp | 6 ++++++ src/Access/LDAPAccessStorage.h | 1 + src/Interpreters/Context.cpp | 2 +- 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 874ae612034..f60a50dc9c3 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -151,6 +151,12 @@ std::vector IAccessStorage::find(EntityType type, const Strings & names) c } +std::optional IAccessStorage::findOrGenerate(EntityType type, const String & name) const +{ + return findOrGenerateImpl(type, name); +} + + UUID IAccessStorage::getID(EntityType type, const String & name) const { auto id = findImpl(type, name); @@ -430,6 +436,12 @@ Poco::Logger * IAccessStorage::getLogger() const } +std::optional IAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const +{ + return findImpl(type, name); +} + + void IAccessStorage::throwNotFound(const UUID & id) const { throw Exception(outputID(id) + " not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 7851f8c9b6b..f6f80dce000 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -48,6 +48,12 @@ public: template std::vector find(const Strings & names) const { return find(EntityClassT::TYPE, names); } + /// Searches for an entity with specified type and name. Returns std::nullopt if not found and cannot be generated. + std::optional findOrGenerate(EntityType type, const String & name) const; + + template + std::optional findOrGenerate(const String & name) const { return findOrGenerate(EntityClassT::TYPE, name); } + /// Searches for an entity with specified name and type. Throws an exception if not found. UUID getID(EntityType type, const String & name) const; @@ -139,6 +145,7 @@ public: protected: virtual std::optional findImpl(EntityType type, const String & name) const = 0; + virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const; virtual std::vector findAllImpl(EntityType type) const = 0; virtual bool existsImpl(const UUID & id) const = 0; virtual AccessEntityPtr readImpl(const UUID & id) const = 0; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 92aef53af64..0df0e7d852b 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -74,6 +74,12 @@ bool LDAPAccessStorage::isStorageReadOnly() const std::optional LDAPAccessStorage::findImpl(EntityType type, const String & name) const +{ + return memory_storage.find(type, name); +} + + +std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const { if (type == EntityType::USER) { diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index eb8b5cac1bc..e570d7a84a8 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -38,6 +38,7 @@ public: // IAccessStorage implementations. private: // IAccessStorage implementations. virtual std::optional findImpl(EntityType type, const String & name) const override; + virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const override; virtual std::vector findAllImpl(EntityType type) const override; virtual bool existsImpl(const UUID & id) const override; virtual AccessEntityPtr readImpl(const UUID & id) const override; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 70cf41a679c..9c4167e6caf 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -673,7 +673,7 @@ void Context::setUser(const String & name, const String & password, const Poco:: client_info.current_password = password; #endif - auto new_user_id = getAccessControlManager().find(name); + auto new_user_id = getAccessControlManager().findOrGenerate(name); std::shared_ptr new_access; if (new_user_id) { From f7e93531f840eb1a044c40851290df20e0765f57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 22:53:49 +0300 Subject: [PATCH 015/441] Simplify init script --- debian/clickhouse-server.init | 69 ++--------------------------------- 1 file changed, 3 insertions(+), 66 deletions(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index b82c70bd6e0..978feb10337 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -160,82 +160,19 @@ initdb() start() { - [ -x $CLICKHOUSE_BINDIR/$PROGRAM ] || exit 0 - local EXIT_STATUS - EXIT_STATUS=0 - - echo -n "Start $PROGRAM service: " - - if is_running; then - echo -n "already running " - EXIT_STATUS=1 - else - ulimit -n 262144 - mkdir -p $CLICKHOUSE_PIDDIR - chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR - initdb - if ! is_running; then - # Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition. - # But clickhouse-server has protection from simultaneous runs with same data directory. - su -s $SHELL ${CLICKHOUSE_USER} -c "$FLOCK -u 9; $CLICKHOUSE_PROGRAM_ENV exec -a \"$PROGRAM\" \"$CLICKHOUSE_BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\"" - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 0 ]; then - return $EXIT_STATUS - fi - fi - fi - - if [ $EXIT_STATUS -eq 0 ]; then - attempts=0 - while ! is_running && [ $attempts -le ${CLICKHOUSE_START_TIMEOUT:=10} ]; do - attempts=$(($attempts + 1)) - sleep 1 - done - if is_running; then - echo "DONE" - else - echo "UNKNOWN" - fi - else - echo "FAILED" - fi - - return $EXIT_STATUS + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" } stop() { - #local EXIT_STATUS - EXIT_STATUS=0 - - if [ -f $CLICKHOUSE_PIDFILE ]; then - - echo -n "Stop $PROGRAM service: " - - kill -TERM $(cat "$CLICKHOUSE_PIDFILE") - - if ! wait_for_done ${CLICKHOUSE_STOP_TIMEOUT}; then - EXIT_STATUS=2 - echo "TIMEOUT" - else - echo "DONE" - fi - - fi - return $EXIT_STATUS + ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}" } restart() { - check_config - if stop; then - if start; then - return 0 - fi - fi - return 1 + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" } From 3d6e56cd613229f635539fefaff7bf29591a4ade Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 27 Aug 2020 00:34:33 +0400 Subject: [PATCH 016/441] Maintain the list and update role changes in cached users --- src/Access/LDAPAccessStorage.cpp | 56 ++++++++++++++++++++++++++++++++ src/Access/LDAPAccessStorage.h | 4 +++ 2 files changed, 60 insertions(+) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 0df0e7d852b..7ec838c56f9 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -52,6 +52,13 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m ldap_server = ldap_server_cfg; roles.swap(roles_cfg); access_control_manager = access_control_manager_; + role_change_subscription = access_control_manager->subscribeForChanges( + [this] (const UUID & id, const AccessEntityPtr & entity) + { + return this->processRoleChange(id, entity); + } + ); + roles_of_interest.clear(); } @@ -61,6 +68,54 @@ bool LDAPAccessStorage::isConfiguredNoLock() const } +void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) +{ + auto role_ptr = typeid_cast>(entity); + if (role_ptr) + { + if (roles.find(role_ptr->getName()) != roles.end()) + { + auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr + { + auto user_ptr = typeid_cast>(cached_entity); + if (user_ptr && !user_ptr->granted_roles.roles.contains(id)) + { + auto clone = user_ptr->clone(); + auto user_clone_ptr = typeid_cast>(clone); + user_clone_ptr->granted_roles.grant(id); + return user_clone_ptr; + } + return cached_entity; + }; + + memory_storage.update(memory_storage.findAll(), update_func); + roles_of_interest.insert(id); + } + } + else + { + if (roles_of_interest.find(id) != roles_of_interest.end()) + { + auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr + { + auto user_ptr = typeid_cast>(cached_entity); + if (user_ptr && user_ptr->granted_roles.roles.contains(id)) + { + auto clone = user_ptr->clone(); + auto user_clone_ptr = typeid_cast>(clone); + user_clone_ptr->granted_roles.revoke(id); + return user_clone_ptr; + } + return cached_entity; + }; + + memory_storage.update(memory_storage.findAll(), update_func); + roles_of_interest.erase(id); + } + } +} + + const char * LDAPAccessStorage::getStorageType() const { return STORAGE_TYPE; @@ -124,6 +179,7 @@ std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const return {}; } + roles_of_interest.insert(role_id.value()); user->granted_roles.grant(role_id.value()); } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index e570d7a84a8..434033bc3c6 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -54,11 +55,14 @@ private: // IAccessStorage implementations. private: bool isConfiguredNoLock() const; + void processRoleChange(const UUID & id, const AccessEntityPtr & entity); mutable std::recursive_mutex mutex; String ldap_server; std::set roles; AccessControlManager * access_control_manager = nullptr; + ext::scope_guard role_change_subscription; + mutable std::set roles_of_interest; mutable MemoryAccessStorage memory_storage; }; } From a4a612843160840b73e4ab5c546ee11a69e52eca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 23:45:24 +0300 Subject: [PATCH 017/441] Trigger CI --- src/Dictionaries/FileDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 18893a99f4e..402fb876c09 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -32,7 +32,7 @@ FileDictionarySource::FileDictionarySource( { const String user_files_path = context.getUserFilesPath(); if (!startsWith(filepath, user_files_path)) - throw Exception("File path " + filepath + " is not inside " + user_files_path, ErrorCodes::PATH_ACCESS_DENIED); + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", filepath, user_files_path); } } From 969940b4c9ba13219531796fbaa3573a8d9ffdbc Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 19 Aug 2020 16:34:38 +0300 Subject: [PATCH 018/441] write table tructure for table function remote(...) --- src/Databases/DatabaseOnDisk.cpp | 8 ++- src/Interpreters/InterpreterCreateQuery.cpp | 13 ++++- src/Parsers/ASTCreateQuery.cpp | 12 ++++- src/Parsers/ParserCreateQuery.cpp | 7 ++- src/Storages/StorageDistributed.cpp | 36 +++---------- src/Storages/StorageDistributed.h | 21 ++------ src/Storages/StorageProxy.h | 60 +++++++++++++++++++++ src/TableFunctions/ITableFunction.cpp | 3 +- src/TableFunctions/ITableFunction.h | 8 ++- src/TableFunctions/TableFunctionRemote.cpp | 58 ++++++++++++++------ src/TableFunctions/TableFunctionRemote.h | 10 ++++ 11 files changed, 166 insertions(+), 70 deletions(-) create mode 100644 src/Storages/StorageProxy.h diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index b69454e3b83..fdfc185f920 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -54,9 +54,13 @@ std::pair createTableFromAST( if (ast_create_query.as_table_function) { - const auto & table_function = ast_create_query.as_table_function->as(); + const auto & ast_table_function = ast_create_query.as_table_function->as(); const auto & factory = TableFunctionFactory::instance(); - StoragePtr storage = factory.get(table_function.name, context)->execute(ast_create_query.as_table_function, context, ast_create_query.table); + auto table_function = factory.get(ast_table_function.name, context); + ColumnsDescription columns; + if (ast_create_query.columns_list && ast_create_query.columns_list->columns) + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, false); + StoragePtr storage = table_function->execute(ast_create_query.as_table_function, context, ast_create_query.table, std::move(columns)); storage->renameInMemory(ast_create_query); return {ast_create_query.table, storage}; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ef81eb41672..dd5fb4d063a 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -456,6 +456,9 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS if (create.columns_list) { + if (create.as_table_function && (create.columns_list->indices || create.columns_list->constraints)) + throw Exception("Indexes and constraints are not supported for table functions", ErrorCodes::INCORRECT_QUERY); + if (create.columns_list->columns) { bool sanity_check_compression_codecs = !create.attach && !context.getSettingsRef().allow_suspicious_codecs; @@ -492,7 +495,13 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS properties.columns = ColumnsDescription(as_select_sample.getNamesAndTypesList()); } else if (create.as_table_function) - return {}; + { + /// Table function without columns list. + auto table_function = TableFunctionFactory::instance().get(create.as_table_function->as().name, context); + properties.columns = table_function->getActualTableStructure(create.as_table_function, context); + if (properties.columns.empty()) //FIXME + return {}; + } else throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); @@ -761,7 +770,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, { const auto & table_function = create.as_table_function->as(); const auto & factory = TableFunctionFactory::instance(); - res = factory.get(table_function.name, context)->execute(create.as_table_function, context, create.table); + res = factory.get(table_function.name, context)->execute(create.as_table_function, context, create.table, properties.columns); res->renameInMemory({create.database, create.table, create.uuid}); } else diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index 73903e28f84..8c5681c3a9c 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -282,13 +282,23 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat if (as_table_function) { + if (columns_list) + { + frame.expression_list_always_start_on_new_line = true; + settings.ostr << (settings.one_line ? " (" : "\n("); + FormatStateStacked frame_nested = frame; + columns_list->formatImpl(settings, state, frame_nested); + settings.ostr << (settings.one_line ? ")" : "\n)"); + frame.expression_list_always_start_on_new_line = false; + } + settings.ostr << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : ""); as_table_function->formatImpl(settings, state, frame); } frame.expression_list_always_start_on_new_line = true; - if (columns_list) + if (columns_list && !as_table_function) { settings.ostr << (settings.one_line ? " (" : "\n("); FormatStateStacked frame_nested = frame; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 55208ca4133..fb0fd20b4ed 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -416,7 +416,12 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return false; if (!storage_p.parse(pos, storage, expected) && !is_temporary) - return false; + { + if (!s_as.ignore(pos, expected)) + return false; + if (!table_function_p.parse(pos, as_table_function, expected)) + return false; + } } else { diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f536c6ee763..fe756e613ac 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -292,12 +292,14 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & storage_policy_name_, const String & relative_data_path_, - bool attach_) + bool attach_, + ClusterPtr owned_cluster_) : IStorage(id_) , remote_database(remote_database_) , remote_table(remote_table_) , global_context(std::make_unique(context_)) , log(&Poco::Logger::get("StorageDistributed (" + id_.table_name + ")")) + , owned_cluster(std::move(owned_cluster_)) , cluster_name(global_context->getMacros()->expand(cluster_name_)) , has_sharding_key(sharding_key_) , relative_data_path(relative_data_path_) @@ -341,40 +343,14 @@ StorageDistributed::StorageDistributed( const ASTPtr & sharding_key_, const String & storage_policy_name_, const String & relative_data_path_, - bool attach) - : StorageDistributed(id_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, storage_policy_name_, relative_data_path_, attach) + bool attach, + ClusterPtr owned_cluster_) + : StorageDistributed(id_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, storage_policy_name_, relative_data_path_, attach, std::move(owned_cluster_)) { remote_table_function_ptr = std::move(remote_table_function_ptr_); } -StoragePtr StorageDistributed::createWithOwnCluster( - const StorageID & table_id_, - const ColumnsDescription & columns_, - const String & remote_database_, /// database on remote servers. - const String & remote_table_, /// The name of the table on the remote servers. - ClusterPtr owned_cluster_, - const Context & context_) -{ - auto res = create(table_id_, columns_, ConstraintsDescription{}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String(), String(), false); - res->owned_cluster = std::move(owned_cluster_); - return res; -} - - -StoragePtr StorageDistributed::createWithOwnCluster( - const StorageID & table_id_, - const ColumnsDescription & columns_, - ASTPtr & remote_table_function_ptr_, - ClusterPtr & owned_cluster_, - const Context & context_) -{ - auto res = create(table_id_, columns_, ConstraintsDescription{}, remote_table_function_ptr_, String{}, context_, ASTPtr(), String(), String(), false); - res->owned_cluster = owned_cluster_; - return res; -} - - bool StorageDistributed::canForceGroupByNoMerge(const Context &context, QueryProcessingStage::Enum to_stage, const ASTPtr & query_ptr) const { const auto & settings = context.getSettingsRef(); diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index b7663676d52..24c07d7e7b7 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -42,21 +42,6 @@ class StorageDistributed final : public ext::shared_ptr_helper +#include + +namespace DB +{ + +template +class StorageTableFunction : public StorageT +{ +public: + using GetStructureFunc = std::function; + + template + StorageTableFunction(GetStructureFunc get_structure_, StorageArgs && ... args) + : StorageT(std::forward(args)...), get_structure(std::move(get_structure_)) + { + } + + Pipe read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) + { + assertBlocksHaveEqualStructure(); + return StorageT::read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + } + + BlockOutputStreamPtr write( + const ASTPtr & query, + const StorageMetadataPtr & metadata_snapshot, + const Context & context) + { + assertBlocksHaveEqualStructure(); + return StorageT::write(query, metadata_snapshot, context); + } + +private: + void assertSourceStructure() + { + if (!get_structure) + return; + + StorageInMemoryMetadata source_metadata; + source_metadata.setColumns(get_structure()); + actual_source_structure = source_metadata.getSampleBlock(); + assertBlocksHaveEqualStructure(getInMemoryMetadataPtr()->getSampleBlock(), actual_source_structure); + + get_structure = {}; + } + + GetStructureFunc get_structure; + Block actual_source_structure; +}; + +} diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 6a784c062da..678f1921385 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -13,10 +13,11 @@ namespace ProfileEvents namespace DB { -StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns_) const { ProfileEvents::increment(ProfileEvents::TableFunctionExecute); context.checkAccess(AccessType::CREATE_TEMPORARY_TABLE | StorageFactory::instance().getSourceAccessType(getStorageTypeName())); + cached_columns = std::move(cached_columns_); return executeImpl(ast_function, context, table_name); } diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index 0bbd7e787a5..09f4d3ca80d 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -31,14 +32,19 @@ public: /// Get the main function name. virtual std::string getName() const = 0; + virtual ColumnsDescription getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) { return {}; } + /// Create storage according to the query. - StoragePtr execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const; + StoragePtr execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns_ = {}) const; virtual ~ITableFunction() {} private: virtual StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const = 0; virtual const char * getStorageTypeName() const = 0; + +protected: + mutable ColumnsDescription cached_columns; }; using TableFunctionPtr = std::shared_ptr; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 7fa3868d142..3dd258f979c 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -27,8 +27,12 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const + +void TableFunctionRemote::prepareClusterInfo(const ASTPtr & ast_function, const Context & context) const { + if (cluster) + return; + ASTs & args_func = ast_function->children; if (args_func.size() != 1) @@ -44,7 +48,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C String cluster_description; String remote_database; String remote_table; - ASTPtr remote_table_function_ptr; + //ASTPtr remote_table_function_ptr; String username; String password; @@ -136,7 +140,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C for (auto ast : args) setIdentifierSpecial(ast); - ClusterPtr cluster; + //ClusterPtr cluster; if (!cluster_name.empty()) { /// Use an existing cluster from the main config @@ -189,30 +193,54 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C if (!remote_table_function_ptr && remote_table.empty()) throw Exception("The name of remote table cannot be empty", ErrorCodes::BAD_ARGUMENTS); - auto remote_table_id = StorageID::createEmpty(); + //auto remote_table_id = StorageID::createEmpty(); remote_table_id.database_name = remote_database; remote_table_id.table_name = remote_table; - auto structure_remote_table = getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); +} + +StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + prepareClusterInfo(ast_function, context); + if (cached_columns.empty()) + cached_columns = getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); + //auto structure_remote_table = getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); StoragePtr res = remote_table_function_ptr - ? StorageDistributed::createWithOwnCluster( + ? StorageDistributed::create( StorageID(getDatabaseName(), table_name), - structure_remote_table, + cached_columns, + ConstraintsDescription{}, remote_table_function_ptr, - cluster, - context) - : StorageDistributed::createWithOwnCluster( + String{}, + context, + ASTPtr{}, + String{}, + String{}, + false, + cluster) + : StorageDistributed::create( StorageID(getDatabaseName(), table_name), - structure_remote_table, - remote_database, - remote_table, - cluster, - context); + cached_columns, + ConstraintsDescription{}, + remote_table_id.database_name, + remote_table_id.table_name, + String{}, + context, + ASTPtr{}, + String{}, + String{}, + false, + cluster); res->startup(); return res; } +ColumnsDescription TableFunctionRemote::getActualTableStructure(const ASTPtr & ast_function, const Context & context) +{ + prepareClusterInfo(ast_function, context); + return getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); +} TableFunctionRemote::TableFunctionRemote(const std::string & name_, bool secure_) : name{name_}, secure{secure_} diff --git a/src/TableFunctions/TableFunctionRemote.h b/src/TableFunctions/TableFunctionRemote.h index 2dd58a8a6a7..27885280920 100644 --- a/src/TableFunctions/TableFunctionRemote.h +++ b/src/TableFunctions/TableFunctionRemote.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include namespace DB @@ -20,14 +22,22 @@ public: std::string getName() const override { return name; } + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) override; + private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Distributed"; } + void prepareClusterInfo(const ASTPtr & ast_function, const Context & context) const; + std::string name; bool is_cluster_function; std::string help_message; bool secure; + + mutable ClusterPtr cluster; + mutable StorageID remote_table_id = StorageID::createEmpty(); + mutable ASTPtr remote_table_function_ptr; }; } From c72765187b5d9f154fcffd4f3f254a75f176a5a0 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 27 Aug 2020 12:36:31 +0400 Subject: [PATCH 019/441] GCC 9 compilation fix --- src/Access/LDAPAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 7ec838c56f9..afd3d6a2eb0 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -78,7 +78,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr { auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && !user_ptr->granted_roles.roles.contains(id)) + if (user_ptr && user_ptr->granted_roles.roles.find(id) == user_ptr->granted_roles.roles.end()) { auto clone = user_ptr->clone(); auto user_clone_ptr = typeid_cast>(clone); @@ -99,7 +99,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr { auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && user_ptr->granted_roles.roles.contains(id)) + if (user_ptr && user_ptr->granted_roles.roles.find(id) != user_ptr->granted_roles.roles.end()) { auto clone = user_ptr->clone(); auto user_clone_ptr = typeid_cast>(clone); From fc1b112e2b3a4cf03af7206c58831fec582b37d2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 27 Aug 2020 21:50:13 +0300 Subject: [PATCH 020/441] Update clickhouse-server.init --- debian/clickhouse-server.init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 978feb10337..4764bca768b 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -160,7 +160,7 @@ initdb() start() { - ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" } @@ -172,7 +172,7 @@ stop() restart() { - ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" } From bce8166420a289feebd14adc8fc3fa831d47da45 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 27 Aug 2020 21:51:19 +0300 Subject: [PATCH 021/441] Update FileDictionarySource.cpp --- src/Dictionaries/FileDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 402fb876c09..82aea4cbb98 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -60,7 +60,7 @@ BlockInputStreamPtr FileDictionarySource::loadAll() std::string FileDictionarySource::toString() const { - return "File: " + filepath + ' ' + format; + return fmt::format("File: {}, {}", filepath, format); } From b521ca9b181be25d5b081b9a2e76f0e6457452a7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 26 Aug 2020 23:56:30 +0300 Subject: [PATCH 022/441] save structure for all table functions --- src/Storages/StorageFile.cpp | 6 +- src/Storages/StorageFile.h | 2 +- src/Storages/StorageGenerateRandom.h | 2 +- src/Storages/StorageHDFS.h | 2 +- src/Storages/StorageInput.h | 2 +- src/Storages/StorageMerge.h | 2 +- src/Storages/StorageMySQL.h | 2 +- ...{StorageProxy.h => StorageTableFunction.h} | 18 ++++- src/Storages/StorageURL.h | 2 +- src/Storages/StorageValues.h | 2 +- src/Storages/StorageXDBC.h | 3 +- src/Storages/System/StorageSystemNumbers.h | 2 +- src/Storages/System/StorageSystemZeros.h | 2 +- src/TableFunctions/ITableFunction.h | 4 +- src/TableFunctions/ITableFunctionFileLike.cpp | 43 ++++++++--- src/TableFunctions/ITableFunctionFileLike.h | 14 +++- src/TableFunctions/ITableFunctionXDBC.cpp | 37 ++++++--- src/TableFunctions/ITableFunctionXDBC.h | 9 +++ src/TableFunctions/TableFunctionFile.cpp | 5 +- src/TableFunctions/TableFunctionFile.h | 3 +- .../TableFunctionGenerateRandom.cpp | 33 ++++++-- .../TableFunctionGenerateRandom.h | 9 +++ src/TableFunctions/TableFunctionHDFS.cpp | 6 +- src/TableFunctions/TableFunctionHDFS.h | 3 +- src/TableFunctions/TableFunctionInput.cpp | 31 +++++++- src/TableFunctions/TableFunctionInput.h | 5 ++ src/TableFunctions/TableFunctionMerge.cpp | 30 ++++++-- src/TableFunctions/TableFunctionMerge.h | 6 ++ src/TableFunctions/TableFunctionMySQL.cpp | 53 ++++++++----- src/TableFunctions/TableFunctionMySQL.h | 20 +++++ src/TableFunctions/TableFunctionNumbers.cpp | 47 +++++++++--- src/TableFunctions/TableFunctionNumbers.h | 6 ++ src/TableFunctions/TableFunctionRemote.cpp | 2 +- src/TableFunctions/TableFunctionRemote.h | 2 +- src/TableFunctions/TableFunctionS3.cpp | 75 ++++++++++--------- src/TableFunctions/TableFunctionS3.h | 20 ++--- src/TableFunctions/TableFunctionURL.cpp | 7 +- src/TableFunctions/TableFunctionURL.h | 3 +- src/TableFunctions/TableFunctionValues.cpp | 33 ++++++-- src/TableFunctions/TableFunctionValues.h | 5 ++ src/TableFunctions/TableFunctionZeros.cpp | 41 +++++++--- src/TableFunctions/TableFunctionZeros.h | 5 ++ 42 files changed, 447 insertions(+), 157 deletions(-) rename src/Storages/{StorageProxy.h => StorageTableFunction.h} (73%) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 558216a6216..6ead7a914ff 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -161,7 +161,11 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us if (args.format_name == "Distributed") { - if (!paths.empty()) + if (paths.empty()) + { + throw Exception("Cannot get table structure from file, because no files match specified name", ErrorCodes::INCORRECT_FILE_NAME); + } + else { auto & first_path = paths[0]; Block header = StorageDistributedDirectoryMonitor::createStreamFromFile(first_path)->getHeader(); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index ea70dcd5311..872a0fc163f 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -18,7 +18,7 @@ namespace DB class StorageFileBlockInputStream; class StorageFileBlockOutputStream; -class StorageFile final : public ext::shared_ptr_helper, public IStorage +class StorageFile : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageGenerateRandom.h b/src/Storages/StorageGenerateRandom.h index e0f037f9a08..09208e9c5cd 100644 --- a/src/Storages/StorageGenerateRandom.h +++ b/src/Storages/StorageGenerateRandom.h @@ -9,7 +9,7 @@ namespace DB { /* Generates random data for given schema. */ -class StorageGenerateRandom final : public ext::shared_ptr_helper, public IStorage +class StorageGenerateRandom : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageHDFS.h b/src/Storages/StorageHDFS.h index fdeaf4ae1b3..ecb77ef83af 100644 --- a/src/Storages/StorageHDFS.h +++ b/src/Storages/StorageHDFS.h @@ -13,7 +13,7 @@ namespace DB * This class represents table engine for external hdfs files. * Read method is supported for now. */ -class StorageHDFS final : public ext::shared_ptr_helper, public IStorage +class StorageHDFS : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageInput.h b/src/Storages/StorageInput.h index c19b19e4703..40b0fd39139 100644 --- a/src/Storages/StorageInput.h +++ b/src/Storages/StorageInput.h @@ -8,7 +8,7 @@ namespace DB /** Internal temporary storage for table function input(...) */ -class StorageInput final : public ext::shared_ptr_helper, public IStorage +class StorageInput : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 5e2d56d18c0..ad898ba2ffd 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -13,7 +13,7 @@ namespace DB /** A table that represents the union of an arbitrary number of other tables. * All tables must have the same structure. */ -class StorageMerge final : public ext::shared_ptr_helper, public IStorage +class StorageMerge : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index a7f98c4379b..fedd8467210 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -20,7 +20,7 @@ namespace DB * Use ENGINE = mysql(host_port, database_name, table_name, user_name, password) * Read only. */ -class StorageMySQL final : public ext::shared_ptr_helper, public IStorage +class StorageMySQL : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageTableFunction.h similarity index 73% rename from src/Storages/StorageProxy.h rename to src/Storages/StorageTableFunction.h index 510368b1d15..5dbbae9d46e 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageTableFunction.h @@ -1,15 +1,22 @@ #pragma once #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int INCOMPATIBLE_COLUMNS; +} + +using GetStructureFunc = std::function; + template class StorageTableFunction : public StorageT { public: - using GetStructureFunc = std::function; template StorageTableFunction(GetStructureFunc get_structure_, StorageArgs && ... args) @@ -17,6 +24,8 @@ public: { } + String getName() const { return "TableFunction" + StorageT::getName(); } + Pipe read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -26,7 +35,7 @@ public: size_t max_block_size, unsigned num_streams) { - assertBlocksHaveEqualStructure(); + assertSourceStructure(); return StorageT::read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); } @@ -35,7 +44,7 @@ public: const StorageMetadataPtr & metadata_snapshot, const Context & context) { - assertBlocksHaveEqualStructure(); + assertSourceStructure(); return StorageT::write(query, metadata_snapshot, context); } @@ -48,7 +57,8 @@ private: StorageInMemoryMetadata source_metadata; source_metadata.setColumns(get_structure()); actual_source_structure = source_metadata.getSampleBlock(); - assertBlocksHaveEqualStructure(getInMemoryMetadataPtr()->getSampleBlock(), actual_source_structure); + if (!blocksHaveEqualStructure(StorageT::getInMemoryMetadataPtr()->getSampleBlock(), actual_source_structure)) + throw Exception("Source storage and table function have different structure", ErrorCodes::INCOMPATIBLE_COLUMNS); get_structure = {}; } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 7983ad71520..69fe19650eb 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -93,7 +93,7 @@ private: BlockOutputStreamPtr writer; }; -class StorageURL final : public ext::shared_ptr_helper, public IStorageURLBase +class StorageURL : public ext::shared_ptr_helper, public IStorageURLBase { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageValues.h b/src/Storages/StorageValues.h index 8a1a06eeb54..b69af140567 100644 --- a/src/Storages/StorageValues.h +++ b/src/Storages/StorageValues.h @@ -9,7 +9,7 @@ namespace DB /* One block storage used for values table function * It's structure is similar to IStorageSystemOneBlock */ -class StorageValues final : public ext::shared_ptr_helper, public IStorage +class StorageValues : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageXDBC.h b/src/Storages/StorageXDBC.h index 508edf22684..7f93cbcd320 100644 --- a/src/Storages/StorageXDBC.h +++ b/src/Storages/StorageXDBC.h @@ -34,6 +34,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; + std::string getName() const override; private: BridgeHelperPtr bridge_helper; @@ -61,8 +62,6 @@ private: size_t max_block_size) const override; Block getHeaderBlock(const Names & column_names, const StorageMetadataPtr & metadata_snapshot) const override; - - std::string getName() const override; }; } diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index f907f3d5f93..cdffd6c8434 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -23,7 +23,7 @@ class Context; * In multithreaded case, if even_distributed is False, implementation with atomic is used, * and result is always in [0 ... limit - 1] range. */ -class StorageSystemNumbers final : public ext::shared_ptr_helper, public IStorage +class StorageSystemNumbers : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/System/StorageSystemZeros.h b/src/Storages/System/StorageSystemZeros.h index 41de3ce6246..26379df5956 100644 --- a/src/Storages/System/StorageSystemZeros.h +++ b/src/Storages/System/StorageSystemZeros.h @@ -14,7 +14,7 @@ namespace DB * You could also specify a limit (how many zeros to give). * If multithreaded is specified, zeros will be generated in several streams. */ -class StorageSystemZeros final : public ext::shared_ptr_helper, public IStorage +class StorageSystemZeros : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index 09f4d3ca80d..2db943225f9 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -24,7 +24,7 @@ class Context; * - go to `example01-01-1`, in `merge` database, `hits` table. */ -class ITableFunction +class ITableFunction : public std::enable_shared_from_this { public: static inline std::string getDatabaseName() { return "_table_function"; } @@ -32,7 +32,7 @@ public: /// Get the main function name. virtual std::string getName() const = 0; - virtual ColumnsDescription getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) { return {}; } + virtual ColumnsDescription getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const { return {}; } /// Create storage according to the query. StoragePtr execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns_ = {}) const; diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 46a64cef785..cf52c6e7163 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -13,7 +13,6 @@ #include #include - namespace DB { @@ -23,8 +22,11 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const Context & context) const { + if (!filename.empty()) + return; + /// Parse args ASTs & args_func = ast_function->children; @@ -39,8 +41,8 @@ StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, cons for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - std::string filename = args[0]->as().value.safeGet(); - std::string format = args[1]->as().value.safeGet(); + filename = args[0]->as().value.safeGet(); + format = args[1]->as().value.safeGet(); if (args.size() == 2 && getName() == "file") { @@ -51,24 +53,41 @@ StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, cons throw Exception("Table function '" + getName() + "' requires 3 or 4 arguments: filename, format, structure and compression method (default auto).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - ColumnsDescription columns; - std::string compression_method = "auto"; - if (args.size() > 2) - { - auto structure = args[2]->as().value.safeGet(); - columns = parseColumnsListFromString(structure, context); - } + structure = args[2]->as().value.safeGet(); if (args.size() == 4) compression_method = args[3]->as().value.safeGet(); +} + +StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; /// Create table - StoragePtr storage = getStorage(filename, format, columns, const_cast(context), table_name, compression_method); + StoragePtr storage = getStorage(filename, format, cached_columns, const_cast(context), table_name, compression_method, std::move(get_structure)); storage->startup(); return storage; } +ColumnsDescription ITableFunctionFileLike::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + if (structure.empty()) + { + assert(getName() == "file" && format == "Distributed"); + return {}; /// TODO get matching path, read structure + } + return parseColumnsListFromString(structure, context); +} + } diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 101f4104cb8..56e5521ceb6 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB { @@ -14,7 +15,18 @@ class ITableFunctionFileLike : public ITableFunction { private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + virtual StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const = 0; + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const = 0; + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String filename; + mutable String format; + mutable String structure; + mutable String compression_method = "auto"; }; } diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index e3b36e97525..ea1f8118ce1 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -28,8 +29,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void ITableFunctionXDBC::parseArguments(const ASTPtr & ast_function, const Context & context) const { + if (helper) + return; + const auto & args_func = ast_function->as(); if (!args_func.arguments) @@ -44,10 +48,6 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - std::string connection_string; - std::string schema_name; - std::string remote_table_name; - if (args.size() == 3) { connection_string = args[0]->as().value.safeGet(); @@ -60,11 +60,16 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co remote_table_name = args[1]->as().value.safeGet(); } - /* Infer external table structure */ /// Have to const_cast, because bridges store their commands inside context - BridgeHelperPtr helper = createBridgeHelper(const_cast(context), context.getSettingsRef().http_receive_timeout.value, connection_string); + helper = createBridgeHelper(const_cast(context), context.getSettingsRef().http_receive_timeout.value, connection_string); helper->startBridgeSync(); +} +ColumnsDescription ITableFunctionXDBC::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + + /* Infer external table structure */ Poco::URI columns_info_uri = helper->getColumnsInfoURI(); columns_info_uri.addQueryParameter("connection_string", connection_string); if (!schema_name.empty()) @@ -73,7 +78,7 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co const auto use_nulls = context.getSettingsRef().external_table_functions_use_nulls; columns_info_uri.addQueryParameter("external_table_functions_use_nulls", - Poco::NumberFormatter::format(use_nulls)); + Poco::NumberFormatter::format(use_nulls)); ReadWriteBufferFromHTTP buf(columns_info_uri, Poco::Net::HTTPRequest::HTTP_POST, {}, ConnectionTimeouts::getHTTPTimeouts(context)); @@ -81,7 +86,21 @@ StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Co readStringBinary(columns_info, buf); NamesAndTypesList columns = NamesAndTypesList::parse(columns_info); - auto result = std::make_shared(StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, ColumnsDescription{columns}, context, helper); + return ColumnsDescription{columns}; +} + +StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + auto result = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, cached_columns, context, helper); if (!result) throw Exception("Failed to instantiate storage from table function " + getName(), ErrorCodes::UNKNOWN_EXCEPTION); diff --git a/src/TableFunctions/ITableFunctionXDBC.h b/src/TableFunctions/ITableFunctionXDBC.h index 547fda3f734..f2b3e70a3a9 100644 --- a/src/TableFunctions/ITableFunctionXDBC.h +++ b/src/TableFunctions/ITableFunctionXDBC.h @@ -24,6 +24,15 @@ private: virtual BridgeHelperPtr createBridgeHelper(Context & context, const Poco::Timespan & http_timeout_, const std::string & connection_string_) const = 0; + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String connection_string; + mutable String schema_name; + mutable String remote_table_name; + mutable BridgeHelperPtr helper; }; class TableFunctionJDBC : public ITableFunctionXDBC diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 4f67f4cfd10..77161d5f02e 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -9,11 +9,12 @@ namespace DB { StoragePtr TableFunctionFile::getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const std::string & compression_method) const + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const std::string & compression_method, GetStructureFunc get_structure) const { StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format, compression_method, columns, ConstraintsDescription{}, global_context}; - return StorageFile::create(source, global_context.getUserFilesPath(), args); + return std::make_shared>(std::move(get_structure), source, global_context.getUserFilesPath(), args); } void registerTableFunctionFile(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionFile.h b/src/TableFunctions/TableFunctionFile.h index e0d8c20ac61..8a8d8d887a8 100644 --- a/src/TableFunctions/TableFunctionFile.h +++ b/src/TableFunctions/TableFunctionFile.h @@ -23,6 +23,7 @@ public: private: StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const std::string & compression_method) const override; + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const std::string & compression_method, GetStructureFunc get_structure) const override; const char * getStorageTypeName() const override { return "File"; } };} diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index 548db38515c..086a76e0ae9 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -13,6 +13,9 @@ #include #include +#include +#include + #include "registerTableFunctions.h" @@ -26,8 +29,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) const { + if (!structure.empty()) + return; + ASTs & args_func = ast_function->children; if (args_func.size() != 1) @@ -58,11 +64,7 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, } /// Parsing first argument as table structure and creating a sample block - std::string structure = args[0]->as().value.safeGet(); - - UInt64 max_string_length = 10; - UInt64 max_array_length = 10; - std::optional random_seed; + structure = args[0]->as().value.safeGet(); if (args.size() >= 2) { @@ -76,11 +78,26 @@ StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, if (args.size() == 4) max_array_length = args[3]->as().value.safeGet(); +} +ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + return parseColumnsListFromString(structure, context); +} - ColumnsDescription columns = parseColumnsListFromString(structure, context); +StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); - auto res = StorageGenerateRandom::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + auto res = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), cached_columns, max_array_length, max_string_length, random_seed); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionGenerateRandom.h b/src/TableFunctions/TableFunctionGenerateRandom.h index b0919608737..45ad93ffee7 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.h +++ b/src/TableFunctions/TableFunctionGenerateRandom.h @@ -16,6 +16,15 @@ public: private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "GenerateRandom"; } + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String structure; + mutable UInt64 max_string_length = 10; + mutable UInt64 max_array_length = 10; + mutable std::optional random_seed; + }; diff --git a/src/TableFunctions/TableFunctionHDFS.cpp b/src/TableFunctions/TableFunctionHDFS.cpp index 3bd6cd3ed76..52397caac67 100644 --- a/src/TableFunctions/TableFunctionHDFS.cpp +++ b/src/TableFunctions/TableFunctionHDFS.cpp @@ -10,9 +10,11 @@ namespace DB { StoragePtr TableFunctionHDFS::getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const { - return StorageHDFS::create(source, + return std::make_shared>(std::move(get_structure), + source, StorageID(getDatabaseName(), table_name), format, columns, diff --git a/src/TableFunctions/TableFunctionHDFS.h b/src/TableFunctions/TableFunctionHDFS.h index 5e8503b318e..91cbfd7c578 100644 --- a/src/TableFunctions/TableFunctionHDFS.h +++ b/src/TableFunctions/TableFunctionHDFS.h @@ -26,7 +26,8 @@ public: private: StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const override; + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const override; const char * getStorageTypeName() const override { return "HDFS"; } }; diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index 5b4a3d97ee4..5b36bb77b2c 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -7,8 +7,10 @@ #include #include #include +#include #include #include +#include #include #include "registerTableFunctions.h" @@ -22,8 +24,11 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -StoragePtr TableFunctionInput::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionInput::parseArguments(const ASTPtr & ast_function, const Context & context) const { + if (!structure.empty()) + return; + const auto * function = ast_function->as(); if (!function->arguments) @@ -35,9 +40,27 @@ StoragePtr TableFunctionInput::executeImpl(const ASTPtr & ast_function, const Co throw Exception("Table function '" + getName() + "' requires exactly 1 argument: structure", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - String structure = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context)->as().value.safeGet(); - auto columns = parseColumnsListFromString(structure, context); - StoragePtr storage = StorageInput::create(StorageID(getDatabaseName(), table_name), columns); + structure = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context)->as().value.safeGet(); +} + +ColumnsDescription TableFunctionInput::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + return parseColumnsListFromString(structure, context); +} + +StoragePtr TableFunctionInput::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + StoragePtr storage = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), cached_columns); storage->startup(); diff --git a/src/TableFunctions/TableFunctionInput.h b/src/TableFunctions/TableFunctionInput.h index a2791533c5d..50744d3ca14 100644 --- a/src/TableFunctions/TableFunctionInput.h +++ b/src/TableFunctions/TableFunctionInput.h @@ -19,6 +19,11 @@ public: private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Input"; } + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String structure; }; } diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 7c0c1fb233f..ffc8076b868 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -45,8 +46,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str return any_table->getInMemoryMetadataPtr()->getColumns().getAllPhysical(); } - -StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionMerge::parseArguments(const ASTPtr & ast_function, const Context & context) const { ASTs & args_func = ast_function->children; @@ -65,12 +65,30 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co args[0] = evaluateConstantExpressionForDatabaseName(args[0], context); args[1] = evaluateConstantExpressionAsLiteral(args[1], context); - String source_database = args[0]->as().value.safeGet(); - String table_name_regexp = args[1]->as().value.safeGet(); + source_database = args[0]->as().value.safeGet(); + table_name_regexp = args[1]->as().value.safeGet(); +} - auto res = StorageMerge::create( +ColumnsDescription TableFunctionMerge::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + return ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}; +} + +StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + auto res = std::make_shared>(std::move(get_structure), StorageID(getDatabaseName(), table_name), - ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}, + cached_columns, source_database, table_name_regexp, context); diff --git a/src/TableFunctions/TableFunctionMerge.h b/src/TableFunctions/TableFunctionMerge.h index b11a9551d34..0e6878983e3 100644 --- a/src/TableFunctions/TableFunctionMerge.h +++ b/src/TableFunctions/TableFunctionMerge.h @@ -18,6 +18,12 @@ public: private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Merge"; } + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String source_database; + mutable String table_name_regexp; }; diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 0afc74f163e..66496778460 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -13,6 +13,7 @@ # include # include # include +# include # include # include # include @@ -21,8 +22,6 @@ # include # include "registerTableFunctions.h" -# include - namespace DB { @@ -35,8 +34,7 @@ namespace ErrorCodes extern const int UNKNOWN_TABLE; } - -StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, const Context & context) const { const auto & args_func = ast_function->as(); @@ -52,14 +50,12 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - std::string host_port = args[0]->as().value.safeGet(); - std::string remote_database_name = args[1]->as().value.safeGet(); - std::string remote_table_name = args[2]->as().value.safeGet(); - std::string user_name = args[3]->as().value.safeGet(); - std::string password = args[4]->as().value.safeGet(); + String host_port = args[0]->as().value.safeGet(); + remote_database_name = args[1]->as().value.safeGet(); + remote_table_name = args[2]->as().value.safeGet(); + user_name = args[3]->as().value.safeGet(); + password = args[4]->as().value.safeGet(); - bool replace_query = false; - std::string on_duplicate_clause; if (args.size() >= 6) replace_query = args[5]->as().value.safeGet() > 0; if (args.size() == 7) @@ -71,9 +67,14 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co ErrorCodes::BAD_ARGUMENTS); /// 3306 is the default MySQL port number - auto parsed_host_port = parseAddress(host_port, 3306); + parsed_host_port = parseAddress(host_port, 3306); +} - mysqlxx::Pool pool(remote_database_name, parsed_host_port.first, user_name, password, parsed_host_port.second); +ColumnsDescription TableFunctionMySQL::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + if (!pool) + pool.emplace(remote_database_name, parsed_host_port.first, user_name, password, parsed_host_port.second); /// Determine table definition by running a query to INFORMATION_SCHEMA. @@ -99,7 +100,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co << " ORDER BY ORDINAL_POSITION"; NamesAndTypesList columns; - MySQLBlockInputStream result(pool.get(), query.str(), sample_block, DEFAULT_BLOCK_SIZE); + MySQLBlockInputStream result(pool->get(), query.str(), sample_block, DEFAULT_BLOCK_SIZE); while (Block block = result.read()) { size_t rows = block.rows(); @@ -117,17 +118,35 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co if (columns.empty()) throw Exception("MySQL table " + backQuoteIfNeed(remote_database_name) + "." + backQuoteIfNeed(remote_table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - auto res = StorageMySQL::create( + return ColumnsDescription{columns}; +} + +StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + if (!pool) + pool.emplace(remote_database_name, parsed_host_port.first, user_name, password, parsed_host_port.second); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + auto res = std::make_shared>(std::move(get_structure), StorageID(getDatabaseName(), table_name), - std::move(pool), + std::move(*pool), remote_database_name, remote_table_name, replace_query, on_duplicate_clause, - ColumnsDescription{columns}, + cached_columns, ConstraintsDescription{}, context); + pool.reset(); + res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionMySQL.h b/src/TableFunctions/TableFunctionMySQL.h index 850affc5887..e8f5851b502 100644 --- a/src/TableFunctions/TableFunctionMySQL.h +++ b/src/TableFunctions/TableFunctionMySQL.h @@ -1,6 +1,11 @@ #pragma once +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif +#if USE_MYSQL #include +#include namespace DB @@ -21,6 +26,21 @@ public: private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "MySQL"; } + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable std::pair parsed_host_port; + mutable String remote_database_name; + mutable String remote_table_name; + mutable String user_name; + mutable String password; + mutable bool replace_query = false; + mutable String on_duplicate_clause; + + mutable std::optional pool; }; } + +#endif diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index c8c0fe96092..256ec47b21d 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -5,8 +5,10 @@ #include #include #include +#include #include #include +#include #include "registerTableFunctions.h" @@ -19,8 +21,10 @@ namespace ErrorCodes } template -StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionNumbers::parseArguments(const ASTPtr & ast_function, const Context & context) const { + + if (const auto * function = ast_function->as()) { auto arguments = function->arguments->children; @@ -28,15 +32,40 @@ StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_f if (arguments.size() != 1 && arguments.size() != 2) throw Exception("Table function '" + getName() + "' requires 'length' or 'offset, length'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; - UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); - - auto res = StorageSystemNumbers::create(StorageID(getDatabaseName(), table_name), multithreaded, length, offset, false); - res->startup(); - return res; + if (arguments.size() == 1) + length = evaluateArgument(context, arguments[0]); + else + { + offset = evaluateArgument(context, arguments[0]); + length = evaluateArgument(context, arguments[1]); + } } - throw Exception("Table function '" + getName() + "' requires 'limit' or 'offset, limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + else + throw Exception("Table function '" + getName() + "' requires 'limit' or 'offset, limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); +} + +template +ColumnsDescription TableFunctionNumbers::getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const +{ + return ColumnsDescription({{"number", std::make_shared()}}); +} + +template +StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + auto res = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), multithreaded, length, offset, false); + res->startup(); + return res; } void registerTableFunctionNumbers(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionNumbers.h b/src/TableFunctions/TableFunctionNumbers.h index c3efbc426ef..5d12eefd21c 100644 --- a/src/TableFunctions/TableFunctionNumbers.h +++ b/src/TableFunctions/TableFunctionNumbers.h @@ -22,6 +22,12 @@ private: const char * getStorageTypeName() const override { return "SystemNumbers"; } UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable UInt64 offset = 0; + mutable UInt64 length; }; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 3dd258f979c..cc5485189d0 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -236,7 +236,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C return res; } -ColumnsDescription TableFunctionRemote::getActualTableStructure(const ASTPtr & ast_function, const Context & context) +ColumnsDescription TableFunctionRemote::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const { prepareClusterInfo(ast_function, context); return getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); diff --git a/src/TableFunctions/TableFunctionRemote.h b/src/TableFunctions/TableFunctionRemote.h index 27885280920..35d1ca3b9ff 100644 --- a/src/TableFunctions/TableFunctionRemote.h +++ b/src/TableFunctions/TableFunctionRemote.h @@ -22,7 +22,7 @@ public: std::string getName() const override { return name; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) override; + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 1a0bff12ba2..4c2779f72d0 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -21,8 +22,10 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionS3::parseArguments(const ASTPtr & ast_function, const Context & context) const { + + /// Parse args ASTs & args_func = ast_function->children; @@ -38,11 +41,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Conte for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); - String filename = args[0]->as().value.safeGet(); - String format; - String structure; - String access_key_id; - String secret_access_key; + filename = args[0]->as().value.safeGet(); if (args.size() < 5) { @@ -57,47 +56,49 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Conte structure = args[4]->as().value.safeGet(); } - String compression_method; if (args.size() == 4 || args.size() == 6) compression_method = args.back()->as().value.safeGet(); - else - compression_method = "auto"; +} - ColumnsDescription columns = parseColumnsListFromString(structure, context); +ColumnsDescription TableFunctionS3::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + return parseColumnsListFromString(structure, context); +} - /// Create table - StoragePtr storage = getStorage(filename, access_key_id, secret_access_key, format, columns, const_cast(context), table_name, compression_method); +StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + Poco::URI uri (filename); + S3::URI s3_uri (uri); + UInt64 min_upload_part_size = context.getSettingsRef().s3_min_upload_part_size; + + StoragePtr storage = std::make_shared>(std::move(get_structure), + s3_uri, + access_key_id, + secret_access_key, + StorageID(getDatabaseName(), table_name), + format, + min_upload_part_size, + cached_columns, + ConstraintsDescription{}, + const_cast(context), + compression_method); storage->startup(); return storage; } -StoragePtr TableFunctionS3::getStorage( - const String & source, - const String & access_key_id, - const String & secret_access_key, - const String & format, - const ColumnsDescription & columns, - Context & global_context, - const std::string & table_name, - const String & compression_method) -{ - Poco::URI uri (source); - S3::URI s3_uri (uri); - UInt64 min_upload_part_size = global_context.getSettingsRef().s3_min_upload_part_size; - return StorageS3::create( - s3_uri, - access_key_id, - secret_access_key, - StorageID(getDatabaseName(), table_name), - format, - min_upload_part_size, - columns, - ConstraintsDescription{}, - global_context, - compression_method); -} void registerTableFunctionS3(TableFunctionFactory & factory) { diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index b66da57d4fa..23dec9a689c 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -29,17 +29,17 @@ protected: const Context & context, const std::string & table_name) const override; - static StoragePtr getStorage( - const String & source, - const String & access_key_id, - const String & secret_access_key, - const String & format, - const ColumnsDescription & columns, - Context & global_context, - const std::string & table_name, - const String & compression_method); - const char * getStorageTypeName() const override { return "S3"; } + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String filename; + mutable String format; + mutable String structure; + mutable String access_key_id; + mutable String secret_access_key; + mutable String compression_method = "auto"; }; class TableFunctionCOS : public TableFunctionS3 diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 59978ae08b1..74cab6e9f00 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -10,10 +10,13 @@ namespace DB { StoragePtr TableFunctionURL::getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const { Poco::URI uri(source); - return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, global_context, compression_method); + return std::make_shared>(std::move(get_structure), uri, + StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, + global_context, compression_method); } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index 366d573bcf4..d69b7ff33a1 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -21,7 +21,8 @@ public: private: StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const override; + const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const override; const char * getStorageTypeName() const override { return "URL"; } }; diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index b4b243416f2..ca1d8b51b22 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -62,8 +63,10 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args } } -StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionValues::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) const { + + ASTs & args_func = ast_function->children; if (args_func.size() != 1) @@ -83,22 +86,42 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const C "Got '{}' instead", getName(), args[0]->formatForErrorMessage()), ErrorCodes::BAD_ARGUMENTS); } - std::string structure = args[0]->as().value.safeGet(); - ColumnsDescription columns = parseColumnsListFromString(structure, context); + structure = args[0]->as().value.safeGet(); +} + +ColumnsDescription TableFunctionValues::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +{ + parseArguments(ast_function, context); + return parseColumnsListFromString(structure, context); +} + +StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; Block sample_block; - for (const auto & name_type : columns.getOrdinary()) + for (const auto & name_type : cached_columns.getOrdinary()) sample_block.insert({ name_type.type->createColumn(), name_type.type, name_type.name }); MutableColumns res_columns = sample_block.cloneEmptyColumns(); + ASTs & args = ast_function->children.at(0)->children; + /// Parsing other arguments as values and inserting them into columns parseAndInsertValues(res_columns, args, sample_block, context); Block res_block = sample_block.cloneWithColumns(std::move(res_columns)); - auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); + auto res = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), cached_columns, res_block); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionValues.h b/src/TableFunctions/TableFunctionValues.h index 3cc3687dab5..9ab2dfb047e 100644 --- a/src/TableFunctions/TableFunctionValues.h +++ b/src/TableFunctions/TableFunctionValues.h @@ -15,6 +15,11 @@ public: private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Values"; } + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable String structure; }; diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp index 13436f04e1c..56413431c37 100644 --- a/src/TableFunctions/TableFunctionZeros.cpp +++ b/src/TableFunctions/TableFunctionZeros.cpp @@ -3,8 +3,9 @@ #include #include #include -#include #include +#include +#include #include #include #include "registerTableFunctions.h" @@ -19,8 +20,10 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } template -StoragePtr TableFunctionZeros::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionZeros::parseArguments(const ASTPtr & ast_function, const Context & context) const { + + if (const auto * function = ast_function->as()) { auto arguments = function->arguments->children; @@ -28,14 +31,34 @@ StoragePtr TableFunctionZeros::executeImpl(const ASTPtr & ast_fun if (arguments.size() != 1) throw Exception("Table function '" + getName() + "' requires 'length'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - UInt64 length = evaluateArgument(context, arguments[0]); - - auto res = StorageSystemZeros::create(StorageID(getDatabaseName(), table_name), multithreaded, length); - res->startup(); - return res; + length = evaluateArgument(context, arguments[0]); } - throw Exception("Table function '" + getName() + "' requires 'limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + else + throw Exception("Table function '" + getName() + "' requires 'limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); +} + +template +ColumnsDescription TableFunctionZeros::getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const +{ + return ColumnsDescription({{"zero", std::make_shared()}}); +} + +template +StoragePtr TableFunctionZeros::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + parseArguments(ast_function, context); + + if (cached_columns.empty()) + cached_columns = getActualTableStructure(ast_function, context); + + auto get_structure = [=, tf = shared_from_this()]() + { + return tf->getActualTableStructure(ast_function, context); + }; + + auto res = std::make_shared>(std::move(get_structure), StorageID(getDatabaseName(), table_name), multithreaded, length); + res->startup(); + return res; } void registerTableFunctionZeros(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionZeros.h b/src/TableFunctions/TableFunctionZeros.h index 71570c23a89..0d00514c6dc 100644 --- a/src/TableFunctions/TableFunctionZeros.h +++ b/src/TableFunctions/TableFunctionZeros.h @@ -22,6 +22,11 @@ private: const char * getStorageTypeName() const override { return "SystemZeros"; } UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; + + ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) const; + + mutable UInt64 length; }; From f91d57adacf7827b53f56593a83b69e3fae0b92c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 28 Aug 2020 12:06:06 +0400 Subject: [PATCH 023/441] Adjust naming --- src/Access/LDAPAccessStorage.cpp | 10 +++++----- src/Access/LDAPAccessStorage.h | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index afd3d6a2eb0..59f061c51a5 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -49,16 +49,16 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m roles_cfg.insert(role_names.begin(), role_names.end()); } - ldap_server = ldap_server_cfg; - roles.swap(roles_cfg); access_control_manager = access_control_manager_; + ldap_server = ldap_server_cfg; + default_role_names.swap(roles_cfg); + roles_of_interest.clear(); role_change_subscription = access_control_manager->subscribeForChanges( [this] (const UUID & id, const AccessEntityPtr & entity) { return this->processRoleChange(id, entity); } ); - roles_of_interest.clear(); } @@ -73,7 +73,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto role_ptr = typeid_cast>(entity); if (role_ptr) { - if (roles.find(role_ptr->getName()) != roles.end()) + if (default_role_names.find(role_ptr->getName()) != default_role_names.end()) { auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr { @@ -163,7 +163,7 @@ std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); - for (const auto& role_name : roles) + for (const auto& role_name : default_role_names) { std::optional role_id; diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 434033bc3c6..d52056fe947 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -58,11 +58,11 @@ private: void processRoleChange(const UUID & id, const AccessEntityPtr & entity); mutable std::recursive_mutex mutex; - String ldap_server; - std::set roles; AccessControlManager * access_control_manager = nullptr; - ext::scope_guard role_change_subscription; + String ldap_server; + std::set default_role_names; mutable std::set roles_of_interest; + ext::scope_guard role_change_subscription; mutable MemoryAccessStorage memory_storage; }; } From 7ffb618f6eb487ef282a9acf0595c3b7eda4b653 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 28 Aug 2020 16:05:08 +0400 Subject: [PATCH 024/441] Add missing proper findOrGenerateImpl() implementation to MultipleAccessStorage class --- src/Access/MultipleAccessStorage.cpp | 17 +++++++++++++++++ src/Access/MultipleAccessStorage.h | 1 + 2 files changed, 18 insertions(+) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index bf711b54d54..7557649601b 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -104,6 +104,23 @@ std::optional MultipleAccessStorage::findImpl(EntityType type, const Strin } +std::optional MultipleAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + { + auto id = storage->findOrGenerate(type, name); + if (id) + { + std::lock_guard lock{mutex}; + ids_cache.set(*id, storage); + return id; + } + } + return {}; +} + + std::vector MultipleAccessStorage::findAllImpl(EntityType type) const { std::vector all_ids; diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 5d01894621f..cf56d71899e 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -35,6 +35,7 @@ public: protected: std::optional findImpl(EntityType type, const String & name) const override; + std::optional findOrGenerateImpl(EntityType type, const String & name) const override; std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; From f0a5f19dae5598e5dd6f019543816ef4e37b016b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 31 Aug 2020 20:45:21 +0300 Subject: [PATCH 025/441] try use another wrapper for table functions --- src/Storages/StorageProxy.h | 158 ++++++++++++++++++ src/Storages/StorageTableFunction.h | 109 ++++++++---- src/TableFunctions/ITableFunction.cpp | 13 +- src/TableFunctions/ITableFunction.h | 4 +- src/TableFunctions/ITableFunctionFileLike.cpp | 14 +- src/TableFunctions/ITableFunctionFileLike.h | 2 +- src/TableFunctions/ITableFunctionXDBC.cpp | 15 +- src/TableFunctions/TableFunctionFile.cpp | 4 +- src/TableFunctions/TableFunctionFile.h | 2 +- .../TableFunctionGenerateRandom.cpp | 11 +- src/TableFunctions/TableFunctionHDFS.cpp | 4 +- src/TableFunctions/TableFunctionHDFS.h | 2 +- src/TableFunctions/TableFunctionInput.cpp | 12 +- src/TableFunctions/TableFunctionMerge.cpp | 12 +- src/TableFunctions/TableFunctionMySQL.cpp | 11 +- src/TableFunctions/TableFunctionNumbers.cpp | 47 ++---- src/TableFunctions/TableFunctionNumbers.h | 4 - src/TableFunctions/TableFunctionRemote.cpp | 7 +- src/TableFunctions/TableFunctionS3.cpp | 12 +- src/TableFunctions/TableFunctionURL.cpp | 5 +- src/TableFunctions/TableFunctionURL.h | 2 +- src/TableFunctions/TableFunctionValues.cpp | 12 +- src/TableFunctions/TableFunctionZeros.cpp | 41 ++--- src/TableFunctions/TableFunctionZeros.h | 3 - 24 files changed, 308 insertions(+), 198 deletions(-) create mode 100644 src/Storages/StorageProxy.h diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h new file mode 100644 index 00000000000..81edcf27c64 --- /dev/null +++ b/src/Storages/StorageProxy.h @@ -0,0 +1,158 @@ +#pragma once + +#include + + +namespace DB +{ + + +class StorageProxy : public IStorage +{ +public: + + StorageProxy(const StorageID & table_id_) : IStorage(table_id_) {} + + virtual StoragePtr getNested() const = 0; + + String getName() const override { return "StorageProxy"; } + + bool isRemote() const override { return getNested()->isRemote(); } + bool isView() const override { return getNested()->isView(); } + bool supportsSampling() const override { return getNested()->supportsSampling(); } + bool supportsFinal() const override { return getNested()->supportsFinal(); } + bool supportsPrewhere() const override { return getNested()->supportsPrewhere(); } + bool supportsReplication() const override { return getNested()->supportsReplication(); } + bool supportsParallelInsert() const override { return getNested()->supportsParallelInsert(); } + bool supportsDeduplication() const override { return getNested()->supportsDeduplication(); } + bool supportsSettings() const override { return getNested()->supportsSettings(); } + bool noPushingToViews() const override { return getNested()->noPushingToViews(); } + bool hasEvenlyDistributedRead() const override { return getNested()->hasEvenlyDistributedRead(); } + + ColumnSizeByName getColumnSizes() const override { return getNested()->getColumnSizes(); } + NamesAndTypesList getVirtuals() const override { return getNested()->getVirtuals(); } + QueryProcessingStage::Enum getQueryProcessingStage(const Context & context, QueryProcessingStage::Enum to_stage, const ASTPtr & ast) const override + { + return getNested()->getQueryProcessingStage(context, to_stage, ast); + } + + BlockInputStreams watch( + const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum & processed_stage, + size_t max_block_size, + unsigned num_streams) override + { + return getNested()->watch(column_names, query_info, context, processed_stage, max_block_size, num_streams); + } + + Pipe read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override + { + return getNested()->read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + } + + BlockOutputStreamPtr write( + const ASTPtr & query, + const StorageMetadataPtr & metadata_snapshot, + const Context & context) override + { + return getNested()->write(query, metadata_snapshot, context); + } + + void drop() override { getNested()->drop(); } + + void truncate( + const ASTPtr & query, + const StorageMetadataPtr & metadata_snapshot, + const Context & context, + TableExclusiveLockHolder & lock) override + { + getNested()->truncate(query, metadata_snapshot, context, lock); + } + + void rename(const String & new_path_to_table_data, const StorageID & new_table_id) override + { + getNested()->rename(new_path_to_table_data, new_table_id); + IStorage::renameInMemory(new_table_id); + } + + void renameInMemory(const StorageID & new_table_id) override + { + getNested()->renameInMemory(new_table_id); + IStorage::renameInMemory(new_table_id); + } + + void alter(const AlterCommands & params, const Context & context, TableLockHolder & alter_lock_holder) override + { + getNested()->alter(params, context, alter_lock_holder); + IStorage::setInMemoryMetadata(getNested()->getInMemoryMetadata()); + } + + void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) const override + { + getNested()->checkAlterIsPossible(commands, settings); + } + + Pipe alterPartition( + const ASTPtr & query, + const StorageMetadataPtr & metadata_snapshot, + const PartitionCommands & commands, + const Context & context) override + { + return getNested()->alterPartition(query, metadata_snapshot, commands, context); + } + + void checkAlterPartitionIsPossible(const PartitionCommands & commands, const StorageMetadataPtr & metadata_snapshot, const Settings & settings) const override + { + getNested()->checkAlterPartitionIsPossible(commands, metadata_snapshot, settings); + } + + bool optimize( + const ASTPtr & query, + const StorageMetadataPtr & metadata_snapshot, + const ASTPtr & partition, + bool final, + bool deduplicate, + const Context & context) override + { + return getNested()->optimize(query, metadata_snapshot, partition, final, deduplicate, context); + } + + void mutate(const MutationCommands & commands, const Context & context) override { getNested()->mutate(commands, context); } + + CancellationCode killMutation(const String & mutation_id) override { return getNested()->killMutation(mutation_id); } + + void startup() override { getNested()->startup(); } + void shutdown() override { getNested()->shutdown(); } + + ActionLock getActionLock(StorageActionBlockType action_type) override { return getNested()->getActionLock(action_type); } + + bool supportsIndexForIn() const override { return getNested()->supportsIndexForIn(); } + bool mayBenefitFromIndexForIn(const ASTPtr & left_in_operand, const Context & query_context, const StorageMetadataPtr & metadata_snapshot) const override + { + return getNested()->mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot); + } + + CheckResults checkData(const ASTPtr & query , const Context & context) override { return getNested()->checkData(query, context); } + void checkTableCanBeDropped() const override { getNested()->checkTableCanBeDropped(); } + void checkPartitionCanBeDropped(const ASTPtr & partition) override { getNested()->checkPartitionCanBeDropped(partition); } + Strings getDataPaths() const override { return getNested()->getDataPaths(); } + StoragePolicyPtr getStoragePolicy() const override { return getNested()->getStoragePolicy(); } + std::optional totalRows() const override { return getNested()->totalRows(); } + std::optional totalBytes() const override { return getNested()->totalBytes(); } + std::optional lifetimeRows() const override { return getNested()->lifetimeRows(); } + std::optional lifetimeBytes() const override { return getNested()->lifetimeBytes(); } + +}; + + +} + diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 5dbbae9d46e..319a2d37dca 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include namespace DB { @@ -11,20 +14,53 @@ namespace ErrorCodes extern const int INCOMPATIBLE_COLUMNS; } -using GetStructureFunc = std::function; +using GetNestedStorageFunc = std::function; -template -class StorageTableFunction : public StorageT +class StorageTableFunctionProxy final : public StorageProxy { public: - - template - StorageTableFunction(GetStructureFunc get_structure_, StorageArgs && ... args) - : StorageT(std::forward(args)...), get_structure(std::move(get_structure_)) + StorageTableFunctionProxy(const StorageID & table_id_, GetNestedStorageFunc get_nested_, ColumnsDescription cached_columns) + : StorageProxy(table_id_), get_nested(std::move(get_nested_)) { + StorageInMemoryMetadata cached_metadata; + cached_metadata.setColumns(std::move(cached_columns)); + setInMemoryMetadata(cached_metadata); } - String getName() const { return "TableFunction" + StorageT::getName(); } + StoragePtr getNested() const override + { + std::lock_guard lock{nested_mutex}; + if (nested) + return nested; + + auto nested_storage = get_nested(); + nested_storage->startup(); + nested = nested_storage; + get_nested = {}; + return nested; + } + + StoragePtr maybeGetNested() const + { + std::lock_guard lock{nested_mutex}; + return nested; + } + + String getName() const override + { + std::lock_guard lock{nested_mutex}; + if (nested) + return nested->getName(); + return StorageProxy::getName(); + } + + void startup() override { } + void shutdown() override + { + auto storage = maybeGetNested(); + if (storage) + storage->shutdown(); + } Pipe read( const Names & column_names, @@ -33,38 +69,51 @@ public: const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, - unsigned num_streams) + unsigned num_streams) override { - assertSourceStructure(); - return StorageT::read(column_names, metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); + auto storage = getNested(); + auto nested_metadata = storage->getInMemoryMetadataPtr(); + auto pipe = storage->read(column_names, nested_metadata, query_info, context, processed_stage, max_block_size, num_streams); + if (!pipe.empty()) + { + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, + metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()), + ConvertingTransform::MatchColumnsMode::Name); + }); + } + return pipe; } BlockOutputStreamPtr write( const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, - const Context & context) + const Context & context) override { - assertSourceStructure(); - return StorageT::write(query, metadata_snapshot, context); + auto storage = getNested(); + auto cached_structure = metadata_snapshot->getSampleBlock(); + auto actual_structure = storage->getInMemoryMetadataPtr()->getSampleBlock(); + if (!blocksHaveEqualStructure(actual_structure, cached_structure)) + { + throw Exception("Source storage and table function have different structure", ErrorCodes::INCOMPATIBLE_COLUMNS); + } + return storage->write(query, metadata_snapshot, context); + } + + void renameInMemory(const StorageID & new_table_id) override + { + if (maybeGetNested()) + StorageProxy::renameInMemory(new_table_id); + else + IStorage::renameInMemory(new_table_id); } private: - void assertSourceStructure() - { - if (!get_structure) - return; - - StorageInMemoryMetadata source_metadata; - source_metadata.setColumns(get_structure()); - actual_source_structure = source_metadata.getSampleBlock(); - if (!blocksHaveEqualStructure(StorageT::getInMemoryMetadataPtr()->getSampleBlock(), actual_source_structure)) - throw Exception("Source storage and table function have different structure", ErrorCodes::INCOMPATIBLE_COLUMNS); - - get_structure = {}; - } - - GetStructureFunc get_structure; - Block actual_source_structure; + mutable std::mutex nested_mutex; + mutable GetNestedStorageFunc get_nested; + mutable StoragePtr nested; }; } diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 678f1921385..d5a78bda08a 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -18,7 +19,17 @@ StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & ProfileEvents::increment(ProfileEvents::TableFunctionExecute); context.checkAccess(AccessType::CREATE_TEMPORARY_TABLE | StorageFactory::instance().getSourceAccessType(getStorageTypeName())); cached_columns = std::move(cached_columns_); - return executeImpl(ast_function, context, table_name); + + bool no_conversion_required = hasStaticStructure() && cached_columns == getActualTableStructure(ast_function, context); + if (cached_columns.empty() || no_conversion_required) + return executeImpl(ast_function, context, table_name); + + auto get_storage = [=, tf = shared_from_this()]() -> StoragePtr + { + return tf->executeImpl(ast_function, context, table_name); + }; + + return std::make_shared(StorageID(getDatabaseName(), table_name), std::move(get_storage), cached_columns); } } diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index 2db943225f9..be1593fcaff 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -32,7 +32,9 @@ public: /// Get the main function name. virtual std::string getName() const = 0; - virtual ColumnsDescription getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const { return {}; } + virtual bool hasStaticStructure() const { return false; } + + virtual ColumnsDescription getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const = 0; /// Create storage according to the query. StoragePtr execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns_ = {}) const; diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index cf52c6e7163..22c537bca2f 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -63,19 +63,9 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const C StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; - - /// Create table - StoragePtr storage = getStorage(filename, format, cached_columns, const_cast(context), table_name, compression_method, std::move(get_structure)); - + auto columns = getActualTableStructure(ast_function, context); + StoragePtr storage = getStorage(filename, format, columns, const_cast(context), table_name, compression_method); storage->startup(); - return storage; } diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 56e5521ceb6..47a41c8d1bb 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -18,7 +18,7 @@ private: virtual StoragePtr getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const = 0; + const std::string & table_name, const String & compression_method) const = 0; ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index ea1f8118ce1..6f1c1f5b2db 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -92,19 +92,8 @@ ColumnsDescription ITableFunctionXDBC::getActualTableStructure(const ASTPtr & as StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; - - auto result = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, cached_columns, context, helper); - - if (!result) - throw Exception("Failed to instantiate storage from table function " + getName(), ErrorCodes::UNKNOWN_EXCEPTION); - + auto columns = getActualTableStructure(ast_function, context); + auto result = std::make_shared(StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, columns, context, helper); result->startup(); return result; } diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 77161d5f02e..2814c2a940b 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -10,11 +10,11 @@ namespace DB { StoragePtr TableFunctionFile::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const std::string & compression_method, GetStructureFunc get_structure) const + const std::string & table_name, const std::string & compression_method) const { StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format, compression_method, columns, ConstraintsDescription{}, global_context}; - return std::make_shared>(std::move(get_structure), source, global_context.getUserFilesPath(), args); + return StorageFile::create(source, global_context.getUserFilesPath(), args); } void registerTableFunctionFile(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionFile.h b/src/TableFunctions/TableFunctionFile.h index 8a8d8d887a8..589b9028d96 100644 --- a/src/TableFunctions/TableFunctionFile.h +++ b/src/TableFunctions/TableFunctionFile.h @@ -24,6 +24,6 @@ public: private: StoragePtr getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const std::string & compression_method, GetStructureFunc get_structure) const override; + const std::string & table_name, const std::string & compression_method) const override; const char * getStorageTypeName() const override { return "File"; } };} diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index 086a76e0ae9..eb741ad0ba1 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -89,15 +89,8 @@ ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(const AS StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; - - auto res = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), cached_columns, max_array_length, max_string_length, random_seed); + auto columns = getActualTableStructure(ast_function, context); + auto res = StorageGenerateRandom::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionHDFS.cpp b/src/TableFunctions/TableFunctionHDFS.cpp index 52397caac67..da4c89b1fd1 100644 --- a/src/TableFunctions/TableFunctionHDFS.cpp +++ b/src/TableFunctions/TableFunctionHDFS.cpp @@ -11,9 +11,9 @@ namespace DB { StoragePtr TableFunctionHDFS::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const + const std::string & table_name, const String & compression_method) const { - return std::make_shared>(std::move(get_structure), + return StorageHDFS::create( source, StorageID(getDatabaseName(), table_name), format, diff --git a/src/TableFunctions/TableFunctionHDFS.h b/src/TableFunctions/TableFunctionHDFS.h index 91cbfd7c578..7e631d1b787 100644 --- a/src/TableFunctions/TableFunctionHDFS.h +++ b/src/TableFunctions/TableFunctionHDFS.h @@ -27,7 +27,7 @@ public: private: StoragePtr getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const override; + const std::string & table_name, const String & compression_method) const override; const char * getStorageTypeName() const override { return "HDFS"; } }; diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index 5b36bb77b2c..4d0f8df55e6 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -52,18 +52,8 @@ ColumnsDescription TableFunctionInput::getActualTableStructure(const ASTPtr & as StoragePtr TableFunctionInput::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; - - StoragePtr storage = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), cached_columns); - + auto storage = StorageInput::create(StorageID(getDatabaseName(), table_name), getActualTableStructure(ast_function, context)); storage->startup(); - return storage; } diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index ffc8076b868..360ec583740 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -78,20 +78,14 @@ ColumnsDescription TableFunctionMerge::getActualTableStructure(const ASTPtr & as StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; - - auto res = std::make_shared>(std::move(get_structure), + auto res = StorageMerge::create( StorageID(getDatabaseName(), table_name), - cached_columns, + getActualTableStructure(ast_function, context), source_database, table_name_regexp, context); + res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 66496778460..bff2842c8c1 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -124,24 +124,19 @@ ColumnsDescription TableFunctionMySQL::getActualTableStructure(const ASTPtr & as StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); if (!pool) pool.emplace(remote_database_name, parsed_host_port.first, user_name, password, parsed_host_port.second); - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; + auto columns = getActualTableStructure(ast_function, context); - auto res = std::make_shared>(std::move(get_structure), + auto res = StorageMySQL::create( StorageID(getDatabaseName(), table_name), std::move(*pool), remote_database_name, remote_table_name, replace_query, on_duplicate_clause, - cached_columns, + columns, ConstraintsDescription{}, context); diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 256ec47b21d..03756b37b07 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -20,29 +20,6 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -template -void TableFunctionNumbers::parseArguments(const ASTPtr & ast_function, const Context & context) const -{ - - - if (const auto * function = ast_function->as()) - { - auto arguments = function->arguments->children; - - if (arguments.size() != 1 && arguments.size() != 2) - throw Exception("Table function '" + getName() + "' requires 'length' or 'offset, length'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - if (arguments.size() == 1) - length = evaluateArgument(context, arguments[0]); - else - { - offset = evaluateArgument(context, arguments[0]); - length = evaluateArgument(context, arguments[1]); - } - } - else - throw Exception("Table function '" + getName() + "' requires 'limit' or 'offset, limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); -} template ColumnsDescription TableFunctionNumbers::getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const @@ -53,19 +30,21 @@ ColumnsDescription TableFunctionNumbers::getActualTableStructure( template StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() + if (const auto * function = ast_function->as()) { - return tf->getActualTableStructure(ast_function, context); - }; + auto arguments = function->arguments->children; - auto res = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), multithreaded, length, offset, false); - res->startup(); - return res; + if (arguments.size() != 1 && arguments.size() != 2) + throw Exception("Table function '" + getName() + "' requires 'length' or 'offset, length'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + UInt64 offset = arguments.size() == 2 ? evaluateArgument(context, arguments[0]) : 0; + UInt64 length = arguments.size() == 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]); + + auto res = StorageSystemNumbers::create(StorageID(getDatabaseName(), table_name), multithreaded, length, offset, false); + res->startup(); + return res; + } + throw Exception("Table function '" + getName() + "' requires 'limit' or 'offset, limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } void registerTableFunctionNumbers(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionNumbers.h b/src/TableFunctions/TableFunctionNumbers.h index 5d12eefd21c..a3c4f6f13cb 100644 --- a/src/TableFunctions/TableFunctionNumbers.h +++ b/src/TableFunctions/TableFunctionNumbers.h @@ -24,10 +24,6 @@ private: UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; - - mutable UInt64 offset = 0; - mutable UInt64 length; }; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index cc5485189d0..3d00d0ad897 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -201,14 +201,11 @@ void TableFunctionRemote::prepareClusterInfo(const ASTPtr & ast_function, const StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { prepareClusterInfo(ast_function, context); - if (cached_columns.empty()) - cached_columns = getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); - //auto structure_remote_table = getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); StoragePtr res = remote_table_function_ptr ? StorageDistributed::create( StorageID(getDatabaseName(), table_name), - cached_columns, + getActualTableStructure(ast_function, context), ConstraintsDescription{}, remote_table_function_ptr, String{}, @@ -220,7 +217,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C cluster) : StorageDistributed::create( StorageID(getDatabaseName(), table_name), - cached_columns, + getActualTableStructure(ast_function, context), ConstraintsDescription{}, remote_table_id.database_name, remote_table_id.table_name, diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 4c2779f72d0..b4fe1baffce 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -70,26 +70,18 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Conte { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; - Poco::URI uri (filename); S3::URI s3_uri (uri); UInt64 min_upload_part_size = context.getSettingsRef().s3_min_upload_part_size; - StoragePtr storage = std::make_shared>(std::move(get_structure), + StoragePtr storage = StorageS3::create( s3_uri, access_key_id, secret_access_key, StorageID(getDatabaseName(), table_name), format, min_upload_part_size, - cached_columns, + getActualTableStructure(ast_function, context), ConstraintsDescription{}, const_cast(context), compression_method); diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 74cab6e9f00..69395c9140e 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -11,11 +11,10 @@ namespace DB { StoragePtr TableFunctionURL::getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const + const std::string & table_name, const String & compression_method) const { Poco::URI uri(source); - return std::make_shared>(std::move(get_structure), uri, - StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, + return StorageURL::create( uri, StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, global_context, compression_method); } diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index d69b7ff33a1..4760e0c70f0 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -22,7 +22,7 @@ public: private: StoragePtr getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method, GetStructureFunc get_structure) const override; + const std::string & table_name, const String & compression_method) const override; const char * getStorageTypeName() const override { return "URL"; } }; diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index ca1d8b51b22..e51e28abff3 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -100,16 +100,10 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const C { parseArguments(ast_function, context); - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() - { - return tf->getActualTableStructure(ast_function, context); - }; + auto columns = getActualTableStructure(ast_function, context); Block sample_block; - for (const auto & name_type : cached_columns.getOrdinary()) + for (const auto & name_type : columns.getOrdinary()) sample_block.insert({ name_type.type->createColumn(), name_type.type, name_type.name }); MutableColumns res_columns = sample_block.cloneEmptyColumns(); @@ -121,7 +115,7 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const C Block res_block = sample_block.cloneWithColumns(std::move(res_columns)); - auto res = std::make_shared>(get_structure, StorageID(getDatabaseName(), table_name), cached_columns, res_block); + auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); return res; } diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp index 56413431c37..b633260861c 100644 --- a/src/TableFunctions/TableFunctionZeros.cpp +++ b/src/TableFunctions/TableFunctionZeros.cpp @@ -19,23 +19,6 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -template -void TableFunctionZeros::parseArguments(const ASTPtr & ast_function, const Context & context) const -{ - - - if (const auto * function = ast_function->as()) - { - auto arguments = function->arguments->children; - - if (arguments.size() != 1) - throw Exception("Table function '" + getName() + "' requires 'length'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - length = evaluateArgument(context, arguments[0]); - } - else - throw Exception("Table function '" + getName() + "' requires 'limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); -} template ColumnsDescription TableFunctionZeros::getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const @@ -46,19 +29,21 @@ ColumnsDescription TableFunctionZeros::getActualTableStructure(co template StoragePtr TableFunctionZeros::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - - if (cached_columns.empty()) - cached_columns = getActualTableStructure(ast_function, context); - - auto get_structure = [=, tf = shared_from_this()]() + if (const auto * function = ast_function->as()) { - return tf->getActualTableStructure(ast_function, context); - }; + auto arguments = function->arguments->children; - auto res = std::make_shared>(std::move(get_structure), StorageID(getDatabaseName(), table_name), multithreaded, length); - res->startup(); - return res; + if (arguments.size() != 1) + throw Exception("Table function '" + getName() + "' requires 'length'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + + UInt64 length = evaluateArgument(context, arguments[0]); + + auto res = StorageSystemZeros::create(StorageID(getDatabaseName(), table_name), multithreaded, length); + res->startup(); + return res; + } + throw Exception("Table function '" + getName() + "' requires 'limit'.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); } void registerTableFunctionZeros(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionZeros.h b/src/TableFunctions/TableFunctionZeros.h index 0d00514c6dc..aba161af4d0 100644 --- a/src/TableFunctions/TableFunctionZeros.h +++ b/src/TableFunctions/TableFunctionZeros.h @@ -24,9 +24,6 @@ private: UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; - - mutable UInt64 length; }; From b147ffcd437d7eaa66567fd4f77358447dded8db Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 16 Jun 2020 12:22:55 +0300 Subject: [PATCH 026/441] encrypt, aes_encrypt_mysql, decrypt, aes_decrypt_mysql functions Functions to encrypt/decrypt any input data with OpenSSL's ciphers with custom key, iv, and add (-gcm mode only). _mysql versions are 100% compatitable with corresponding MySQL functions Supported modes depend on OpenSSL version, but generally are: aes-{128,192,56}-{ecb,cbc,cfb1,cfb8,cfb128,ofb,gcm} Please note that in a -gcm mode a 16-byte tag is appended to the ciphertext on encryption and is expected to be found at the end of ciphertext on decryption. Added tests that verify compatibility with MySQL functions, and test vectors for GCM mode from OpenSSL. Added masking rules for aes_X funtions Rules are installed by default to config.d/query_masking_rules.xml --- base/common/defines.h | 2 + cmake/sanitize.cmake | 4 +- debian/clickhouse-server.install | 1 + programs/server/CMakeLists.txt | 2 + .../server/config.d/query_masking_rules.xml | 9 + src/Columns/ColumnString.cpp | 6 + src/Columns/ColumnString.h | 3 + src/Functions/FunctionHelpers.cpp | 2 +- src/Functions/FunctionsAES.cpp | 56 ++ src/Functions/FunctionsAES.h | 656 ++++++++++++++++++ src/Functions/aes_decrypt_mysql.cpp | 29 + src/Functions/aes_encrypt_mysql.cpp | 29 + src/Functions/decrypt.cpp | 29 + src/Functions/encrypt.cpp | 29 + src/Functions/registerFunctions.cpp | 19 + src/Functions/ya.make | 5 + .../0_stateless/01318_decrypt.reference | 80 +++ tests/queries/0_stateless/01318_decrypt.sql | 123 ++++ .../0_stateless/01318_encrypt.reference | 86 +++ tests/queries/0_stateless/01318_encrypt.sql | 125 ++++ tests/ubsan_suppressions.txt | 3 + 21 files changed, 1295 insertions(+), 3 deletions(-) create mode 100644 src/Functions/FunctionsAES.cpp create mode 100644 src/Functions/FunctionsAES.h create mode 100644 src/Functions/aes_decrypt_mysql.cpp create mode 100644 src/Functions/aes_encrypt_mysql.cpp create mode 100644 src/Functions/decrypt.cpp create mode 100644 src/Functions/encrypt.cpp create mode 100644 tests/queries/0_stateless/01318_decrypt.reference create mode 100644 tests/queries/0_stateless/01318_decrypt.sql create mode 100644 tests/queries/0_stateless/01318_encrypt.reference create mode 100644 tests/queries/0_stateless/01318_encrypt.sql create mode 100644 tests/ubsan_suppressions.txt diff --git a/base/common/defines.h b/base/common/defines.h index af5981023ff..722a1f206bf 100644 --- a/base/common/defines.h +++ b/base/common/defines.h @@ -70,10 +70,12 @@ # define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) # define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address"))) # define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread"))) +# define NO_SANITIZE_MEMORY __attribute__((__no_sanitize__("memory"))) #else /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it. # define NO_SANITIZE_UNDEFINED # define NO_SANITIZE_ADDRESS # define NO_SANITIZE_THREAD +# define NO_SANITIZE_MEMORY #endif #if defined __GNUC__ && !defined __clang__ diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 32443ed78c3..043268ceba5 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -48,8 +48,8 @@ if (SANITIZE) endif () elseif (SANITIZE STREQUAL "undefined") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt") if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") endif() diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install index b1475fdf162..57f858407af 100644 --- a/debian/clickhouse-server.install +++ b/debian/clickhouse-server.install @@ -2,5 +2,6 @@ usr/bin/clickhouse-server usr/bin/clickhouse-copier usr/bin/clickhouse-report etc/clickhouse-server/config.xml +etc/clickhouse-server/config.d/*.xml etc/clickhouse-server/users.xml etc/systemd/system/clickhouse-server.service diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 5500a4680b7..adaddcf1762 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -29,6 +29,8 @@ set (CLICKHOUSE_SERVER_LINK clickhouse_program_add(server) install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) +install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) +install(FILES config.d/query_masking_rules.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server/config.d COMPONENT clickhouse) # TODO We actually need this on Mac, FreeBSD. if (OS_LINUX) diff --git a/programs/server/config.d/query_masking_rules.xml b/programs/server/config.d/query_masking_rules.xml index f919523472c..9ba77198d88 100644 --- a/programs/server/config.d/query_masking_rules.xml +++ b/programs/server/config.d/query_masking_rules.xml @@ -15,5 +15,14 @@ TOPSECRET.TOPSECRET [hidden] + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 6c84107caae..d38008c44c7 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -612,4 +612,10 @@ void ColumnString::protect() getOffsets().protect(); } +void ColumnString::validate() const +{ + if (!offsets.empty() && offsets.back() != chars.size()) + throw Exception(fmt::format("ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size()), ErrorCodes::LOGICAL_ERROR); +} + } diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 128e1efe146..19398e07b83 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -267,6 +267,9 @@ public: Offsets & getOffsets() { return offsets; } const Offsets & getOffsets() const { return offsets; } + + // Throws an exception if offsets/chars are messed up + void validate() const; }; diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 18e5fde5462..91c9d67ea2b 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -137,7 +137,7 @@ void validateArgumentsImpl(const IFunction & func, const auto & arg = arguments[i + argument_offset]; const auto descriptor = descriptors[i]; if (int error_code = descriptor.isValid(arg.type, arg.column); error_code != 0) - throw Exception("Illegal type of argument #" + std::to_string(i) + throw Exception("Illegal type of argument #" + std::to_string(argument_offset + i + 1) // +1 is for human-friendly 1-based indexing + (descriptor.argument_name ? " '" + std::string(descriptor.argument_name) + "'" : String{}) + " of function " + func.getName() + (descriptor.expected_type_description ? String(", expected ") + descriptor.expected_type_description : String{}) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp new file mode 100644 index 00000000000..48533be054a --- /dev/null +++ b/src/Functions/FunctionsAES.cpp @@ -0,0 +1,56 @@ +#include + +#if USE_SSL + +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int OPENSSL_ERROR; +} +} + +namespace OpenSSLDetails +{ +void onError(std::string error_message) +{ + error_message += ". OpenSSL error code: " + std::to_string(ERR_get_error()); + throw DB::Exception(error_message, DB::ErrorCodes::OPENSSL_ERROR); +} + +StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key) +{ + memcpy(folded_key.data(), key.data, cipher_key_size); + + for (size_t i = cipher_key_size; i < key.size; ++i) + { + folded_key[i % cipher_key_size] ^= key.data[i]; + } + + return StringRef(folded_key.data(), cipher_key_size); +} + +const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) +{ + const auto *evp_cipher = EVP_get_cipherbyname(cipher_name.data); + if (evp_cipher == nullptr) + { + // For some reasons following ciphers can't be found by name. + if (cipher_name == "aes-128-cfb128") + evp_cipher = EVP_aes_128_cfb128(); + else if (cipher_name == "aes-192-cfb128") + evp_cipher = EVP_aes_192_cfb128(); + else if (cipher_name == "aes-256-cfb128") + evp_cipher = EVP_aes_256_cfb128(); + } + + return evp_cipher; +} + +} + +#endif diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h new file mode 100644 index 00000000000..8d062e9b12d --- /dev/null +++ b/src/Functions/FunctionsAES.h @@ -0,0 +1,656 @@ +#pragma once + +#include + +#if USE_SSL +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} +} + +namespace OpenSSLDetails +{ +[[noreturn]] void onError(std::string error_message); +StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); +const EVP_CIPHER * getCipherByName(const StringRef & name); + +enum class CompatibilityMode +{ + MySQL, + OpenSSL +}; + +enum class CipherMode +{ + MySQLCompatibility, // with key folding + OpenSSLCompatibility, // just as regular openssl's enc application does (AEAD modes, like GCM and CCM are not supported) + RFC5116_AEAD_AES_GCM // AEAD GCM with custom IV length and tag (HMAC) appended to the ciphertext, see https://tools.ietf.org/html/rfc5116#section-5.1 +}; + + +template +struct KeyHolder +{ + inline StringRef setKey(size_t cipher_key_size, const StringRef & key) const + { + if (key.size != cipher_key_size) + throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size), + DB::ErrorCodes::BAD_ARGUMENTS); + + return key; + } +}; + +template <> +struct KeyHolder +{ + inline StringRef setKey(size_t cipher_key_size, const StringRef & key) + { + if (key.size < cipher_key_size) + throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size), + DB::ErrorCodes::BAD_ARGUMENTS); + + // MySQL does something fancy with the keys that are too long, + // ruining compatibility with OpenSSL and not improving security. + // But we have to do the same to be compatitable with MySQL. + // see https://github.com/mysql/mysql-server/blob/8.0/router/src/harness/src/my_aes_openssl.cc#L71 + // (my_aes_create_key function) + return foldEncryptionKeyInMySQLCompatitableMode(cipher_key_size, key, folded_key); + } + + ~KeyHolder() + { + OPENSSL_cleanse(folded_key.data(), folded_key.size()); + } + +private: + std::array folded_key; +}; + +template +inline void validateCipherMode(const EVP_CIPHER * evp_cipher) +{ + if constexpr (compatibility_mode == CompatibilityMode::MySQL) + { + switch (EVP_CIPHER_mode(evp_cipher)) + { + case EVP_CIPH_ECB_MODE: [[fallthrough]]; + case EVP_CIPH_CBC_MODE: [[fallthrough]]; + case EVP_CIPH_CFB_MODE: [[fallthrough]]; + case EVP_CIPH_OFB_MODE: + return; + } + } + else if constexpr (compatibility_mode == CompatibilityMode::OpenSSL) + { + switch (EVP_CIPHER_mode(evp_cipher)) + { + case EVP_CIPH_ECB_MODE: [[fallthrough]]; + case EVP_CIPH_CBC_MODE: [[fallthrough]]; + case EVP_CIPH_CFB_MODE: [[fallthrough]]; + case EVP_CIPH_OFB_MODE: [[fallthrough]]; + case EVP_CIPH_CTR_MODE: [[fallthrough]]; + case EVP_CIPH_GCM_MODE: + return; + } + } + + throw DB::Exception("Unsupported cipher mode " + std::string(EVP_CIPHER_name(evp_cipher)), DB::ErrorCodes::BAD_ARGUMENTS); +} + +template +inline void validateIV(const StringRef & iv_value, const size_t cipher_iv_size) +{ + // In MySQL mode we don't care if IV is longer than expected, only if shorter. + if ((mode == CipherMode::MySQLCompatibility && iv_value.size != 0 && iv_value.size < cipher_iv_size) + || (mode == CipherMode::OpenSSLCompatibility && iv_value.size != 0 && iv_value.size != cipher_iv_size)) + throw DB::Exception(fmt::format("Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size), + DB::ErrorCodes::BAD_ARGUMENTS); +} + +} + +namespace DB +{ +template +class FunctionEncrypt : public IFunction +{ +public: + static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode; + static constexpr auto name = Impl::name; + static FunctionPtr create(const Context &) { return std::make_shared(); } + +private: + using CipherMode = OpenSSLDetails::CipherMode; + + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto optional_args = FunctionArgumentDescriptors{ + {"IV", isStringOrFixedString, nullptr, "Initialization vector binary string"}, + }; + + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL) + { + optional_args.emplace_back(FunctionArgumentDescriptor{ + "AAD", isStringOrFixedString, nullptr, "Additional authenticated data binary string for GCM mode" + }); + } + + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{ + {"mode", isStringOrFixedString, isColumnConst, "encryption mode string"}, + {"input", nullptr, nullptr, "plaintext"}, + {"key", isStringOrFixedString, nullptr, "encryption key binary string"}, + }, + optional_args + ); + + return std::make_shared(); + } + + void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override + { + block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + { + using namespace OpenSSLDetails; + + const auto mode = block.getByPosition(arguments[0]).column->getDataAt(0); + + if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + const auto * evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); + + const auto input_column = block.getByPosition(arguments[1]).column; + const auto key_column = block.getByPosition(arguments[2]).column; + + OpenSSLDetails::validateCipherMode(evp_cipher); + + ColumnPtr result_column; + if (arguments.size() <= 3) + result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr); + else + { + const auto iv_column = block.getByPosition(arguments[3]).column; + if (compatibility_mode != OpenSSLDetails::CompatibilityMode::MySQL && EVP_CIPHER_iv_length(evp_cipher) == 0) + throw Exception(mode.toString() + " does not support IV", ErrorCodes::BAD_ARGUMENTS); + + if (arguments.size() <= 4) + { + result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, nullptr); + } + else + { + if (cipher_mode != EVP_CIPH_GCM_MODE) + throw Exception("AAD can be only set for GCM-mode", ErrorCodes::BAD_ARGUMENTS); + + const auto aad_column = block.getByPosition(arguments[4]).column; + result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + block.getByPosition(result).column = std::move(result_column); + } + + template + static ColumnPtr doEncrypt(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + const IvColumnType & iv_column, + const AadColumnType & aad_column) + { + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::MySQL) + { + return doEncryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + if (EVP_CIPHER_mode(evp_cipher) == EVP_CIPH_GCM_MODE) + { + return doEncryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + return doEncryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + return nullptr; + } + + template + static ColumnPtr doEncryptImpl(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + [[maybe_unused]] const IvColumnType & iv_column, + [[maybe_unused]] const AadColumnType & aad_column) + { + using namespace OpenSSLDetails; + + auto evp_ctx_ptr = std::unique_ptr(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free); + auto evp_ctx = evp_ctx_ptr.get(); + + const auto block_size = static_cast(EVP_CIPHER_block_size(evp_cipher)); + const auto key_size = static_cast(EVP_CIPHER_key_length(evp_cipher)); + [[maybe_unused]] const auto iv_size = static_cast(EVP_CIPHER_iv_length(evp_cipher)); + const auto tag_size = 16; // https://tools.ietf.org/html/rfc5116#section-5.1 + + auto encrypted_result_column = ColumnString::create(); + auto & encrypted_result_column_data = encrypted_result_column->getChars(); + auto & encrypted_result_column_offsets = encrypted_result_column->getOffsets(); + + { + size_t resulting_size = 0; + // for modes with block_size > 1, plaintext is padded up to a block_size, + // which may result in allocating to much for block_size = 1. + // That may lead later to reading unallocated data from underlying PaddedPODArray + // due to assumption that it is safe to read up to 15 bytes past end. + const auto pad_to_next_block = block_size == 1 ? 0 : 1; + for (size_t r = 0; r < input_rows_count; ++r) + { + resulting_size += (input_column->getDataAt(r).size / block_size + pad_to_next_block) * block_size + 1; + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + resulting_size += tag_size; + } +#if defined(MEMORY_SANITIZER) + encrypted_result_column_data.resize_fill(resulting_size, 0xFF); +#else + encrypted_result_column_data.resize(resulting_size); +#endif + } + + auto encrypted = encrypted_result_column_data.data(); + + KeyHolder key_holder; + + for (size_t r = 0; r < input_rows_count; ++r) + { + const auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r)); + auto iv_value = StringRef{}; + if constexpr (!std::is_same_v>) + { + iv_value = iv_column->getDataAt(r); + } + + const auto input_value = input_column->getDataAt(r); + + // 1: Init CTX + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + // 1.a.1: Init CTX with custom IV length and optionally with AAD + if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) + onError("Failed to initialize encryption context with cipher"); + + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1) + onError("Failed to set custom IV length to " + std::to_string(iv_value.size)); + + if (EVP_EncryptInit_ex(evp_ctx, nullptr, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to set key and IV"); + + // 1.a.2 Set AAD + if constexpr (!std::is_same_v>) + { + const auto aad_data = aad_column->getDataAt(r); + int tmp_len = 0; + if (aad_data.size != 0 && EVP_EncryptUpdate(evp_ctx, nullptr, &tmp_len, + reinterpret_cast(aad_data.data), aad_data.size) != 1) + onError("Failed to set AAD data"); + } + } + else + { + // 1.b: Init CTX + validateIV(iv_value, iv_size); + + if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to initialize cipher context"); + } + + int output_len = 0; + // 2: Feed the data to the cipher + if (EVP_EncryptUpdate(evp_ctx, + reinterpret_cast(encrypted), &output_len, + reinterpret_cast(input_value.data), static_cast(input_value.size)) != 1) + onError("Failed to encrypt"); + encrypted += output_len; + + // 3: retrieve encrypted data (ciphertext) + if (EVP_EncryptFinal_ex(evp_ctx, + reinterpret_cast(encrypted), &output_len) != 1) + onError("Failed to fetch ciphertext"); + encrypted += output_len; + + // 4: optionally retrieve a tag and append it to the ciphertext (RFC5116): + // https://tools.ietf.org/html/rfc5116#section-5.1 + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_GET_TAG, tag_size, encrypted) != 1) + onError("Failed to retrieve GCM tag"); + encrypted += tag_size; + } + *encrypted = '\0'; + ++encrypted; + + encrypted_result_column_offsets.push_back(encrypted - encrypted_result_column_data.data()); + + if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) + onError("Failed to reset context"); + } + + // in case of block size of 1, we overestimate buffer required for encrypted data, fix it up. + if (!encrypted_result_column_offsets.empty() && encrypted_result_column_data.size() > encrypted_result_column_offsets.back()) + { + encrypted_result_column_data.resize(encrypted_result_column_offsets.back()); + } + + encrypted_result_column->validate(); + return encrypted_result_column; + } +}; + + +/// AES_decrypt(string, key, block_mode[, init_vector]) +template +class FunctionDecrypt : public IFunction +{ +public: + static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode; + static constexpr auto name = Impl::name; + static FunctionPtr create(const Context &) { return std::make_shared(); } + +private: + using CipherMode = OpenSSLDetails::CipherMode; + + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto optional_args = FunctionArgumentDescriptors{ + {"IV", isStringOrFixedString, nullptr, "Initialization vector binary string"}, + }; + + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL) + { + optional_args.emplace_back(FunctionArgumentDescriptor{ + "AAD", isStringOrFixedString, nullptr, "Additional authenticated data binary string for GCM mode" + }); + } + + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{ + {"mode", isStringOrFixedString, isColumnConst, "decryption mode string"}, + {"input", nullptr, nullptr, "ciphertext"}, + {"key", isStringOrFixedString, nullptr, "decryption key binary string"}, + }, + optional_args + ); + + return std::make_shared(); + } + + void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override + { + block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + { + using namespace OpenSSLDetails; + + const auto mode = block.getByPosition(arguments[0]).column->getDataAt(0); + + if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + const auto * evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + OpenSSLDetails::validateCipherMode(evp_cipher); + + const auto input_column = block.getByPosition(arguments[1]).column; + const auto key_column = block.getByPosition(arguments[2]).column; + + ColumnPtr result_column; + if (arguments.size() <= 3) + result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr); + else + { + const auto iv_column = block.getByPosition(arguments[3]).column; + if (compatibility_mode != OpenSSLDetails::CompatibilityMode::MySQL && EVP_CIPHER_iv_length(evp_cipher) == 0) + throw Exception(mode.toString() + " does not support IV", ErrorCodes::BAD_ARGUMENTS); + + if (arguments.size() <= 4) + { + result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, nullptr); + } + else + { + if (EVP_CIPHER_mode(evp_cipher) != EVP_CIPH_GCM_MODE) + throw Exception("AAD can be only set for GCM-mode", ErrorCodes::BAD_ARGUMENTS); + + const auto aad_column = block.getByPosition(arguments[4]).column; + result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + block.getByPosition(result).column = std::move(result_column); + } + + template + static ColumnPtr doDecrypt(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + const IvColumnType & iv_column, + const AadColumnType & aad_column) + { + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::MySQL) + { + return doDecryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); + if (cipher_mode == EVP_CIPH_GCM_MODE) + { + return doDecryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + return doDecryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + return nullptr; + } + + template + static ColumnPtr doDecryptImpl(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + [[maybe_unused]] const IvColumnType & iv_column, + [[maybe_unused]] const AadColumnType & aad_column) + { + using namespace OpenSSLDetails; + + auto evp_ctx_ptr = std::unique_ptr(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free); + auto evp_ctx = evp_ctx_ptr.get(); + + [[maybe_unused]] const auto block_size = static_cast(EVP_CIPHER_block_size(evp_cipher)); + [[maybe_unused]] const auto iv_size = static_cast(EVP_CIPHER_iv_length(evp_cipher)); + const auto key_size = static_cast(EVP_CIPHER_key_length(evp_cipher)); + const auto tag_size = 16; // https://tools.ietf.org/html/rfc5116#section-5.1 + + auto decrypted_result_column = ColumnString::create(); + auto & decrypted_result_column_data = decrypted_result_column->getChars(); + auto & decrypted_result_column_offsets = decrypted_result_column->getOffsets(); + + { + size_t resulting_size = 0; + for (size_t r = 0; r < input_rows_count; ++r) + { + resulting_size += input_column->getDataAt(r).size + 1; + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + resulting_size -= tag_size; + } + +#if defined(MEMORY_SANITIZER) + // Pre-fill result column with values to prevent MSAN from dropping dead on + // aes-X-ecb mode with "WARNING: MemorySanitizer: use-of-uninitialized-value". + // This is most likely to be caused by the underlying assembler implementation: + // see crypto/aes/aesni-x86_64.s, function aesni_ecb_encrypt + // which msan seems to fail instrument correctly. + decrypted_result_column_data.resize_fill(resulting_size, 0xFF); +#else + decrypted_result_column_data.resize(resulting_size); +#endif + } + auto decrypted = decrypted_result_column_data.data(); + + KeyHolder key_holder; + for (size_t r = 0; r < input_rows_count; ++r) + { + // 0: prepare key if required + auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r)); + auto iv_value = StringRef{}; + if constexpr (!std::is_same_v>) + { + iv_value = iv_column->getDataAt(r); + } + + auto input_value = input_column->getDataAt(r); + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + // empty plaintext results in empty ciphertext + tag, means there should be atleast tag_size bytes. + if (input_value.size < tag_size) + throw Exception(fmt::format("Encrypted data is too short: only {} bytes, " + "should contain at least {} bytes of a tag.", + input_value.size, block_size, tag_size), ErrorCodes::BAD_ARGUMENTS); + input_value.size -= tag_size; + } + + // 1: Init CTX + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + // 1.a.1 : Init CTX with custom IV length and optionally with AAD + if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) + onError("Failed to initialize cipher context"); + + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1) + onError("Failed to set custom IV length to " + std::to_string(iv_value.size)); + + if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to set key and IV"); + + // 1.a.2: Set AAD if present + if constexpr (!std::is_same_v>) + { + const auto aad_data = aad_column->getDataAt(r); + int tmp_len = 0; + if (aad_data.size != 0 && EVP_DecryptUpdate(evp_ctx, nullptr, &tmp_len, + reinterpret_cast(aad_data.data), aad_data.size) != 1) + onError("Failed to sed AAD data"); + } + } + else + { + // 1.b: Init CTX + validateIV(iv_value, iv_size); + + if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to initialize cipher context"); + } + + // 2: Feed the data to the cipher + int output_len = 0; + if (EVP_DecryptUpdate(evp_ctx, + reinterpret_cast(decrypted), &output_len, + reinterpret_cast(input_value.data), static_cast(input_value.size)) != 1) + onError("Failed to decrypt"); + decrypted += output_len; + + // 3: optionally get tag from the ciphertext (RFC5116) and feed it to the context + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + void * tag = const_cast(reinterpret_cast(input_value.data + input_value.size)); + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_TAG, tag_size, tag) != 1) + onError("Failed to set tag"); + } + + // 4: retrieve encrypted data (ciphertext) + if (EVP_DecryptFinal_ex(evp_ctx, + reinterpret_cast(decrypted), &output_len) != 1) + onError("Failed to decrypt"); + decrypted += output_len; + + *decrypted = '\0'; + ++decrypted; + + decrypted_result_column_offsets.push_back(decrypted - decrypted_result_column_data.data()); + + if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) + onError("Failed to reset context"); + } + + // in case we overestimate buffer required for decrypted data, fix it up. + if (!decrypted_result_column_offsets.empty() && decrypted_result_column_data.size() > decrypted_result_column_offsets.back()) + { + decrypted_result_column_data.resize(decrypted_result_column_offsets.back()); + } + + decrypted_result_column->validate(); + return decrypted_result_column; + } +}; + +} + + +#endif diff --git a/src/Functions/aes_decrypt_mysql.cpp b/src/Functions/aes_decrypt_mysql.cpp new file mode 100644 index 00000000000..764fcf06c1a --- /dev/null +++ b/src/Functions/aes_decrypt_mysql.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct DecryptMySQLModeImpl +{ + static constexpr auto name = "aes_decrypt_mysql"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::MySQL; +}; + +} + +namespace DB +{ + +void registerFunctionAESDecryptMysql(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp new file mode 100644 index 00000000000..1d84824d9d6 --- /dev/null +++ b/src/Functions/aes_encrypt_mysql.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct EncryptMySQLModeImpl +{ + static constexpr auto name = "aes_encrypt_mysql"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::MySQL; +}; + +} + +namespace DB +{ + +void registerFunctionAESEncryptMysql(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/decrypt.cpp b/src/Functions/decrypt.cpp new file mode 100644 index 00000000000..1cbda0dba99 --- /dev/null +++ b/src/Functions/decrypt.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct DecryptImpl +{ + static constexpr auto name = "decrypt"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::OpenSSL; +}; + +} + +namespace DB +{ + +void registerFunctionDecrypt(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/encrypt.cpp b/src/Functions/encrypt.cpp new file mode 100644 index 00000000000..807443a2fb8 --- /dev/null +++ b/src/Functions/encrypt.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct EncryptImpl +{ + static constexpr auto name = "encrypt"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::OpenSSL; +}; + +} + +namespace DB +{ + +void registerFunctionEncrypt(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 804ceb4d309..b7fb3cecd66 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -1,3 +1,5 @@ +#include + #include @@ -38,10 +40,19 @@ void registerFunctionsNull(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); void registerFunctionsConsistentHashing(FunctionFactory & factory); void registerFunctionsUnixTimestamp64(FunctionFactory & factory); + #if !defined(ARCADIA_BUILD) void registerFunctionBayesAB(FunctionFactory &); #endif +#if USE_SSL +void registerFunctionEncrypt(FunctionFactory & factory); +void registerFunctionDecrypt(FunctionFactory & factory); +void registerFunctionAESEncryptMysql(FunctionFactory & factory); +void registerFunctionAESDecryptMysql(FunctionFactory & factory); + +#endif + void registerFunctions() { @@ -83,9 +94,17 @@ void registerFunctions() registerFunctionsIntrospection(factory); registerFunctionsConsistentHashing(factory); registerFunctionsUnixTimestamp64(factory); + #if !defined(ARCADIA_BUILD) registerFunctionBayesAB(factory); #endif + +#if USE_SSL + registerFunctionEncrypt(factory); + registerFunctionDecrypt(factory); + registerFunctionAESEncryptMysql(factory); + registerFunctionAESDecryptMysql(factory); +#endif } } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 31d5dfa9fd3..1806d762d33 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -46,6 +46,8 @@ SRCS( addSeconds.cpp addWeeks.cpp addYears.cpp + aes_decrypt_mysql.cpp + aes_encrypt_mysql.cpp appendTrailingCharIfAbsent.cpp array/arrayAll.cpp array/arrayAUC.cpp @@ -138,6 +140,7 @@ SRCS( currentUser.cpp dateDiff.cpp date_trunc.cpp + decrypt.cpp defaultValueOfArgumentType.cpp defaultValueOfTypeName.cpp demange.cpp @@ -145,6 +148,7 @@ SRCS( dumpColumnStructure.cpp e.cpp empty.cpp + encrypt.cpp endsWith.cpp equals.cpp erfc.cpp @@ -170,6 +174,7 @@ SRCS( FunctionFQDN.cpp FunctionHelpers.cpp FunctionJoinGet.cpp + FunctionsAES.cpp FunctionsCoding.cpp FunctionsConversion.cpp FunctionsEmbeddedDictionaries.cpp diff --git a/tests/queries/0_stateless/01318_decrypt.reference b/tests/queries/0_stateless/01318_decrypt.reference new file mode 100644 index 00000000000..2241501f564 --- /dev/null +++ b/tests/queries/0_stateless/01318_decrypt.reference @@ -0,0 +1,80 @@ +0 +0 +0 +1 +MySQL-specific key folding and decrpyting +aes-128-ecb 1 +aes-128-ecb 1 +aes-128-ecb 1 +aes-192-ecb 1 +aes-192-ecb 1 +aes-192-ecb 1 +aes-256-ecb 1 +aes-256-ecb 1 +aes-256-ecb 1 +aes-128-cbc 1 +aes-128-cbc 1 +aes-128-cbc 1 +aes-192-cbc 1 +aes-192-cbc 1 +aes-192-cbc 1 +aes-256-cbc 1 +aes-256-cbc 1 +aes-256-cbc 1 +aes-128-cfb1 1 +aes-128-cfb1 1 +aes-128-cfb1 1 +aes-192-cfb1 1 +aes-192-cfb1 1 +aes-192-cfb1 1 +aes-256-cfb1 1 +aes-256-cfb1 1 +aes-256-cfb1 1 +aes-128-cfb8 1 +aes-128-cfb8 1 +aes-128-cfb8 1 +aes-192-cfb8 1 +aes-192-cfb8 1 +aes-192-cfb8 1 +aes-256-cfb8 1 +aes-256-cfb8 1 +aes-256-cfb8 1 +aes-128-cfb128 1 +aes-128-cfb128 1 +aes-128-cfb128 1 +aes-192-cfb128 1 +aes-192-cfb128 1 +aes-192-cfb128 1 +aes-256-cfb128 1 +aes-256-cfb128 1 +aes-256-cfb128 1 +aes-128-ofb 1 +aes-128-ofb 1 +aes-128-ofb 1 +aes-192-ofb 1 +aes-192-ofb 1 +aes-192-ofb 1 +aes-256-ofb 1 +aes-256-ofb 1 +aes-256-ofb 1 +GCM mode with IV +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +GCM mode with IV and AAD +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +F56E87055BC32D0EEB31B2EACC2BF2A5 1 diff --git a/tests/queries/0_stateless/01318_decrypt.sql b/tests/queries/0_stateless/01318_decrypt.sql new file mode 100644 index 00000000000..5e148b90724 --- /dev/null +++ b/tests/queries/0_stateless/01318_decrypt.sql @@ -0,0 +1,123 @@ +--- aes_decrypt_mysql(string, key, block_mode[, init_vector, AAD]) +-- The MySQL-compatitable encryption, only ecb, cbc, cfb1, cfb8, cfb128 and ofb modes are supported, +-- just like for MySQL +-- https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt +-- https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_block_encryption_mode +-- Please note that for keys that exceed mode-specific length, keys are folded in a MySQL-specific way, +-- meaning that whole key is used, but effective key length is still determined by mode. +-- when key doesn't exceed the default mode length, ecryption result equals with AES_encypt() + +----------------------------------------------------------------------------------------- +-- error cases +----------------------------------------------------------------------------------------- +SELECT aes_decrypt_mysql(); --{serverError 42} not enough arguments +SELECT aes_decrypt_mysql('aes-128-ecb'); --{serverError 42} not enough arguments +SELECT aes_decrypt_mysql('aes-128-ecb', 'text'); --{serverError 42} not enough arguments + + +-- Mode +SELECT aes_decrypt_mysql(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT aes_decrypt_mysql('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT aes_decrypt_mysql('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name +SELECT aes_decrypt_mysql('aes-128-gcm', 'text', 'key'); -- {serverError 36} mode is not supported by _mysql-functions + +SELECT decrypt(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT decrypt('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT decrypt('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name + + +-- Key +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short + +SELECT decrypt('aes-128-ecb', 'text'); --{serverError 42} key is missing +SELECT decrypt('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT decrypt('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short +SELECT decrypt('aes-128-ecb', 'text', 'keykeykeykeykeykeykeykeykeykeykeykey'); -- {serverError 36} key is to long + + +-- IV +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key', 1011); --{serverError 43} bad IV type 6 +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key', 'iv'); --{serverError 36} IV is too short 4 + +SELECT decrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 1011); --{serverError 43} bad IV type 1 +SELECT decrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iviviviviviviviviviviviviviviviviviviviviv'); --{serverError 36} IV is too long 3 +SELECT decrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iv'); --{serverError 36} IV is too short 2 + + +--AAD +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 42} too many arguments + +SELECT decrypt('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type +SELECT decrypt('aes-128-gcm', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type + + +-- decrypting invalid cipher text, should cause an error or produce garbage +SELECT ignore(decrypt('aes-128-ecb', 'hello there', '1111111111111111')); -- {serverError 454} 1 +SELECT ignore(decrypt('aes-128-cbc', 'hello there', '1111111111111111')); -- {serverError 454} 2 +SELECT ignore(decrypt('aes-128-cfb1', 'hello there', '1111111111111111')); -- GIGO +SELECT ignore(decrypt('aes-128-ofb', 'hello there', '1111111111111111')); -- GIGO +SELECT ignore(decrypt('aes-128-ctr', 'hello there', '1111111111111111')); -- GIGO +SELECT decrypt('aes-128-ctr', '', '1111111111111111') == ''; + + +----------------------------------------------------------------------------------------- +-- Validate against predefined ciphertext,plaintext,key and IV for MySQL compatibility mode +----------------------------------------------------------------------------------------- +CREATE TABLE encryption_test +( + input String, + key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), + iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85') +) Engine = Memory; + +INSERT INTO encryption_test (input) +VALUES (''), ('text'), ('What Is ClickHouse? ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).'); + +SELECT 'MySQL-specific key folding and decrpyting'; +SELECT 'aes-128-ecb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-ecb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-ecb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cbc' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cbc' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cbc' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cfb1' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cfb1' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cfb1' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cfb8' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cfb8' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cfb8' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cfb128' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cfb128' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cfb128' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-ofb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-ofb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-ofb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'GCM mode with IV'; +SELECT 'aes-128-gcm' as mode, hex(substr(key, 1, 16)) as key, decrypt(mode, encrypt(mode, input, unhex(key), iv), unhex(key), iv) == input FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(substr(key, 1, 24)) as key, decrypt(mode, encrypt(mode, input, unhex(key), iv), unhex(key), iv) == input FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(substr(key, 1, 32)) as key, decrypt(mode, encrypt(mode, input, unhex(key), iv), unhex(key), iv) == input FROM encryption_test; + +SELECT 'GCM mode with IV and AAD'; +SELECT 'aes-128-gcm' as mode, hex(substr(key, 1, 16)) AS key, decrypt(mode, encrypt(mode, input, unhex(key), iv, 'AAD'), unhex(key), iv, 'AAD') == input FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(substr(key, 1, 24)) AS key, decrypt(mode, encrypt(mode, input, unhex(key), iv, 'AAD'), unhex(key), iv, 'AAD') == input FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(substr(key, 1, 32)) AS key, decrypt(mode, encrypt(mode, input, unhex(key), iv, 'AAD'), unhex(key), iv, 'AAD') == input FROM encryption_test; + + +-- based on https://github.com/openssl/openssl/blob/master/demos/evp/aesgcm.c#L20 +WITH + unhex('eebc1f57487f51921c0465665f8ae6d1658bb26de6f8a069a3520293a572078f') as key, + unhex('67ba0510262ae487d737ee6298f77e0c') as tag, + unhex('99aa3e68ed8173a0eed06684') as iv, + unhex('f56e87055bc32d0eeb31b2eacc2bf2a5') as plaintext, + unhex('4d23c3cec334b49bdb370c437fec78de') as aad, + unhex('f7264413a84c0e7cd536867eb9f21736') as ciphertext +SELECT + hex(decrypt('aes-256-gcm', concat(ciphertext, tag), key, iv, aad)) as plaintext_actual, + plaintext_actual = hex(plaintext); diff --git a/tests/queries/0_stateless/01318_encrypt.reference b/tests/queries/0_stateless/01318_encrypt.reference new file mode 100644 index 00000000000..130a95ab178 --- /dev/null +++ b/tests/queries/0_stateless/01318_encrypt.reference @@ -0,0 +1,86 @@ +UInt64 +5417DEA8D67A1A03FD561809C62402FF +Float64 +9B66D0AA685DC0F1EFFA2E385F7EA2F2 +Decimal64 +5417DEA8D67A1A03FD561809C62402FF +MySQL-specific key folding +aes-128-ecb 861BA71B647390651F75F0EB4A18DCA1 +aes-128-ecb 557F8E81CBBBB2515A33500768018C3C +aes-128-ecb C508CC3350317F61AB1793DB6D93AEFAB000F51C8651ABB578F5EEF362F8560FB3655364CEC9B4D2758C71BC03E4D72FBC54385094A20E20949F70D91462442C5ABA90EF581BC3309C7F2E9E468E34D83C73346C05627D4E1634615F6F5B01E1B388664AECCAD26E4508B537082CEA572222DDBFC9BD0CB5D1D6FEE26A8CCD57BAE76655DCAF0952B80C7F1903990B60 +aes-192-ecb 04793D6184FFAD00C6457B54D30FED91 +aes-192-ecb 0028EDF20F6C08FD7097653CE4DB9C39 +aes-192-ecb 733ECCEEBD7C760CA38FC8ED736A053DCCA5C8DE06035689C765BE53DBFEB9BA9B98C9169C884278E227A3BAFA38F53E837857DF96D6A9B09C58BD196553FFDF1B6F191EAF5A82950EDCEDBE46B91BB8FDC8DDAA6566481B807FA5FCA40D687CF14E2F1A318E0B4CE5C2305FB43A411B4B65EC5B525FD4AB08CDDE49212FC2E99B1096EA5B5F4957594654CA3B369145 +aes-256-ecb 3FEBF71206304655B6451E02EBFDB965 +aes-256-ecb EBB295D0F05E820370629399AD7B04DB +aes-256-ecb 54D9B7BF0FEE21A848051927FB29D9F621FDD4DEA6D320D80D3102E46A92F17D7E2B9B7AB3C0C4B34B1A7ABABDF98E7ACC4066DFCC385AC799A8D686655311886152E49D3AF8F0D4EF321E05E22E3CE19D0CDCA1C05035C86C6EA4D2D2C7B31AA0D496E03CEB7661905F9463A140E5F8875E876CBD1A72A2B4DE0F98533E1C87D06FE4A68ADF572DD9A74A562DE9A45F +aes-128-cbc 9617622E3D3A2BB45B3D0922D63F7C1E +aes-128-cbc 700AED2DCC265D7E8D98D0DBBD7653C4 +aes-128-cbc 63A26A3E2FC9DD48E3BA6CD6BF94D3196181BF85D43301DD7E129C95C90A6760F762317A5E868FECB48CCC9F789B2552A40D12D8B8D8DF062115968751BFD36281D47D63EA4A1629337A0EC5051128FECFE573D6EA055175C17C39C79AF5ECAEB4764ED2AF89784C5BF9F43C75AA4781E5DD483DDCD529D8D30C22710CA930F79BBACBDA51086B227F7A3B087D4EBD7F +aes-192-cbc ABF263E5090CC373E7D92CAE91A9136C +aes-192-cbc B3DBB188BC9DEF8AF436D68A23FEAA99 +aes-192-cbc 99E393B279CB125B11F830262800D00A2E4CEFD59A2D1441AAEC11334DDD2AD2CCE75900BA42BE1A78F95C79CEEA85CB0FA401463A53229F8B520E6883011445AE90C2A3A1ECBC2589E1429E631474B5B269AA7617CB818384E93B51C1B3A73F9183CA27899B41BE3E9BB95A45F70C0BA94331E3B7E4849808F83195979079DAC69953C7208D00D6C2C4E6304CDA6B9E +aes-256-cbc 221820CEBE1F8B472AC06C8C0DE52BA7 +aes-256-cbc ADC9060184FE5A23B6CE35327CE5309A +aes-256-cbc 09E8AE34C02AB3A50BF4FC49A70344C6B956FCA52D764287B3E935D192D80D7F3E22DDA0E42D4911796A3E4D47A4CD884C511D7AEEF89AD739F4A8A519F03C70C5FE0C0C65A8916B3BA1A64D6964693999C002B96CDE2D53327D2614D8D8D473D6F7B537DC5B459F694196ECF0F034156DBB0A91E98735531E5E756908390F262456D104393F099934E1F8E5406D537E +aes-128-cfb1 +aes-128-cfb1 6B939C8E +aes-128-cfb1 50CACD46A945BD61615CFA5638EB313236AE7611B3EA6653E844D7D9F3D4D05442492390DD33D1DA753AD94F6C49D43EB40CF096734AC36D95D1FB6FEDB1AC9D1C0A59C7688C5B5C3D992F6F3FF06418E4E4F5372E51B22F9B7C155AB2B763CD47F5EA5C5F1454D912589DB8E24D46BFE1D429D6F57EEFAFCA91D9C2513705913B3205BFFDA952F65C +aes-192-cfb1 +aes-192-cfb1 B90F814C +aes-192-cfb1 8A3503A6DA7547D161B5F778492910321B570C795450FDF2B67DD571467745397769B919CF00ADA1FFBF9DEEFBA4170E625F2E0F3B371DABF5D9439FB154E09890AB951D18F7C1D2D041E697E6AB1779112C31F068AD59A4E4D9ABF30CA5504174EE62BCA04C092C29C60B29BB633DB31F111F905E33D63418A5048256711EF6A276C8C2417EF94C45 +aes-256-cfb1 +aes-256-cfb1 88876A00 +aes-256-cfb1 A31D52BADDE4E79FA12A3C6E145C86E0DDA653EACFDC934541E4D4E2166A06E801A5BC7D30C430B65BE5F8AF5E8BE0A900F03A54BD16D8CD27BBA3411BA00B10A53FEEF7691398BCE0FFB548A4F856219364DD44C4BD9E5515809018045CBC5CFA5308B25498B9F16E437F10EF6E38F08FDBE8424481B18147690A7A63BE674DB566356A1A6FCD642F +aes-128-cfb8 +aes-128-cfb8 76698980 +aes-128-cfb8 5505B55E6BD7BB69C38FFC9953E1729111D99EB2A6F6392BC1B211141B455923D25FC9387A92C95D91448AA46FD08B0C8977A0CF0123D4503681760D2BAC1203EABB6D6BCD52A9DD99ECD69FA28648229E8F1BC7D194DB817BF39CEC2880722915A542B098FBDE142206A3418B6024B7961BB42358FDB9CB7FC742E387353C961AEB2240F7ABA6EC29 +aes-192-cfb8 +aes-192-cfb8 FB065B88 +aes-192-cfb8 D8015BD1B2CBA6EA04D8A299E17E1D7D0DEE7D31B20711EDF9CEACB158CDDFE4ED569B5C6425DAF6FB0B2B0CA1C87D18C867D87AC760FDD3CF4E94A9FDF1325593F6C36600F4105FEFF5E9B733AB7B28908376DCF4C6FA5F90B720071958C8C69FCABCE9D4D97B33E4A902A41B6F5C72387ADC9F2FD9B787FC0E09FB603CD74BE25DE276D140A00C28 +aes-256-cfb8 +aes-256-cfb8 A62542C4 +aes-256-cfb8 85406B1C10E1F8D7208BD274FD166558C541FF6B761B59BB64BB74A253FE8FE9C6F59BAB49314B2D286270CCC05E6C6C569EB279558E9218162B5CC7D11ECE21CE8CD5805B85F0879EE277AB742C0D07C08A3CA037EAA8D5C643E2E9116D574E9E2284FE7D1EE123640E5F698ACEB407BD8855741472E0BECE67B3760CA2C9F3EB656A6A884AB09C8C +aes-128-cfb128 +aes-128-cfb128 761CC7B1 +aes-128-cfb128 5511DEB1CD27BED6062453F99912A690AA3628279677D02473DDA04CB6763AF2C5DD191F388E16AC7E122D1A070DB9BEE68B403FAB2FBEF09B224C646EDE9A773E2671D8D8B89B263FDD218FE5E8E6FB37CCA8AEC51C8B4BE0BA5FBC1496B95731E405D14B269CEFCAF7244CE893F9F8B950A66BD30540F0F1E1BF6ECB3ABB71F6470B4875A38C5372 +aes-192-cfb128 +aes-192-cfb128 FBD2EB05 +aes-192-cfb128 D8DFF205334E1E67A0FBC960069219C7D75497A6FCD47E98AB572823BCB1CC472FB94A502AD597B565021F78EAFF3BD7D8B4E4C175A76F97B039FB53E73F35B997EBB693A0AB42AA1003946F84FFBEB32F4E1AC57312C05C8F4E315F6275C097862F347CD3B20B56BFFD73A95FC441AEA0DCFB9E3EABE231114C013A769ADA1D0E46A76D4979A27B08 +aes-256-cfb128 +aes-256-cfb128 A6DE9438 +aes-256-cfb128 85D38D383998E41817837668832DA0068BB5B64BB612FFF9C87D8E5773375C6503B50105A6F96F462A62E1E6C5C7FD7081C629818C10E91A23DA38F1421247B83AE2DEBE317C2F221122DB2E6222DC4C274EEFDE5E425D1FCAD20A3B6C82EF3363335F423F9C1E88AE69F017D53F4ADE0FD748B0BDFF7F15400E27E641FC674EBC40B38EF44D911C8B +aes-128-ofb +aes-128-ofb 761CC7B1 +aes-128-ofb 5511DEB1CD27BED6062453F99912A6905E263DE7ABC54E8CF73A9D9FB2F05F643D7E15B468FFF70EB7EFF3A4DD69A60725852D39D4C389231FDD8B82AC16374F101D34D231113E8ACA9762529B38B49313D8700C4650933C3A4E2CE624C0253AEE0ADC8BCB9E6042D1EE5BA75E2327A5D5B039EA4DA20076E8CDFE0E597AD18710DAFC4B261BC02E32 +aes-192-ofb +aes-192-ofb FBD2EB05 +aes-192-ofb D8DFF205334E1E67A0FBC960069219C7F891621DCBF5B36DE32C2BDC0F37DF021B9106560DBEC7FDE953CC5DAA05C5FD2E37EF1ABD955DD77407CF631BFCBE33873E4F1365E316C707F63DE258C0077E403BD2E8F567601F0D5A5B01920E790B358F2B6B73D9BCBFFBCF718C4223A004B46CBB4BA70E31B542263C0902E03A8EF8FA00ACA70770050A +aes-256-ofb +aes-256-ofb A6DE9438 +aes-256-ofb 85D38D383998E41817837668832DA00685DA2D37602CB00AD4DAFB8EB0B85840BCC0CDAD6E229ED356EB2D1E8819E9951D4F6D0F6EA4894F3FD5401938529F388077AC5373CA5D383510769C08256551E6718B236FE29D27C284BB1A3B4E0F6AC5A7625607668F05ED92E7FF54B02771D5ED2AA1381D427E64010EDF21E11CDCDB9B8C35B542720430 +Nullable and LowCardinality +Nullable(String) \N +Nullable(String) A6DE9438 +LowCardinality(String) A6DE9438 +GCM mode with IV +aes-128-gcm 3D67D2B8D8F49A24C482085FEC494231 +aes-128-gcm C08B1CF60C5A2C92C55DAC62223CBA22C736446C +aes-128-gcm E38605F61AE032292F25A35A6CDF6E827980625B9A50BB3C77CD2AD54DB9BE5578322CC55569D1B0C5B82577060C0053388F511DB7BF9C898BF4B05FB6C8C0D0F50449C06A2E89E086610CB2BAEF25B206312836884DCBCC6CD8329B2A43E2BA751183B1696AB3F070BE94FA823F1E1A5E2372A06E1AD2719AF37983D23FCD199820B7769E72EDC20AF48466DAEB6550DC7FDBA041F77D5231 +aes-192-gcm FC2C8C63A570E584AB71F19BA6E79A8F +aes-192-gcm 9A6CF0FDA93F1614F982B5570CC1216D84E356BD +aes-192-gcm B961E9FD9B940EBAD7ADDA75C9F198A40797A598AC7FA183AC58705EF6E4E295504D71B85D81978B4CE196AFFFA019B941F44B14DF06375688FCA986F2F3088C24E955392F0DB378F033052822D560CD8DDFF5472C66029E997AE2D63935DAAA10D6703E5AB627E8168F16CF5CDB1112DD2D49F10E087BA20831DCCE592465C95AAA5AF8F766BAEDC3FD3949EDD2E667333C83E58786504137 +aes-256-gcm E99DBEBC01F021758352D7FBD9039EFA +aes-256-gcm 8742CE3A7B0595B281C712600D274CA881F47414 +aes-256-gcm A44FD73ACEB1A64BDE2D03808A2576EDBB6076F61614CC84A960CCBE55FBABF365671B7017BC89C8A2E0A633E0A05E40B2681B33AD3E7A0AC4925DBD9735C4D1C1E33726B1D6A83CBD337A65C50D7CC33CC4E64369D54C1B6AF3A82D206DF0698BEB61EF9AB2DF81B03DF3829A2EC42D667D87376B8A1351C69BB7A11CCBE50DA88ABA991E98D3BD71129682F35422AD73B05EC624357E77FC +GCM mode with IV and AAD +aes-128-gcm 5AB059BB98F087E8134B19E7EB5CD9C7 +aes-128-gcm C08B1CF67AD5D38FE0F3508689794961B8D1FAB8 +aes-128-gcm E38605F61AE032292F25A35A6CDF6E827980625B9A50BB3C77CD2AD54DB9BE5578322CC55569D1B0C5B82577060C0053388F511DB7BF9C898BF4B05FB6C8C0D0F50449C06A2E89E086610CB2BAEF25B206312836884DCBCC6CD8329B2A43E2BA751183B1696AB3F070BE94FA823F1E1A5E2372A06E1AD2719AF37983D23FCD199820B7769E72EDC20A0826DB2A479DB59F7216A9BDCBD0C989 +aes-192-gcm 04C13E4B1D62481ED22B3644595CB5DB +aes-192-gcm 9A6CF0FD2B329B04EAD18301818F016DF8F77447 +aes-192-gcm B961E9FD9B940EBAD7ADDA75C9F198A40797A598AC7FA183AC58705EF6E4E295504D71B85D81978B4CE196AFFFA019B941F44B14DF06375688FCA986F2F3088C24E955392F0DB378F033052822D560CD8DDFF5472C66029E997AE2D63935DAAA10D6703E5AB627E8168F16CF5CDB1112DD2D49F10E087BA20831DCCE592465C95AAA5AF8F766BAEDC3668E035498D8C46FB662833CCC12C9D6 +aes-256-gcm E94F5F6ED4A99B741D492D7EA10B7147 +aes-256-gcm 8742CE3A3EA5153952DB4C0D94B501FE878FF9A7 +aes-256-gcm A44FD73ACEB1A64BDE2D03808A2576EDBB6076F61614CC84A960CCBE55FBABF365671B7017BC89C8A2E0A633E0A05E40B2681B33AD3E7A0AC4925DBD9735C4D1C1E33726B1D6A83CBD337A65C50D7CC33CC4E64369D54C1B6AF3A82D206DF0698BEB61EF9AB2DF81B03DF3829A2EC42D667D87376B8A1351C69BB7A11CCBE50DA88ABA991E98D3BD712F56268961DDAB59FA4D2B50578602C4 +F7264413A84C0E7CD536867EB9F2173667BA0510262AE487D737EE6298F77E0C 1 diff --git a/tests/queries/0_stateless/01318_encrypt.sql b/tests/queries/0_stateless/01318_encrypt.sql new file mode 100644 index 00000000000..dd5561efda6 --- /dev/null +++ b/tests/queries/0_stateless/01318_encrypt.sql @@ -0,0 +1,125 @@ +--- aes_encrypt_mysql(string, key, block_mode[, init_vector, AAD]) +-- The MySQL-compatitable encryption, only ecb, cbc, cfb1, cfb8, cfb128 and ofb modes are supported, +-- just like for MySQL +-- https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt +-- https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_block_encryption_mode +-- Please note that for keys that exceed mode-specific length, keys are folded in a MySQL-specific way, +-- meaning that whole key is used, but effective key length is still determined by mode. +-- when key doesn't exceed the default mode length, ecryption result equals with AES_encypt() + +----------------------------------------------------------------------------------------- +-- error cases +----------------------------------------------------------------------------------------- +SELECT aes_encrypt_mysql(); --{serverError 42} not enough arguments +SELECT aes_encrypt_mysql('aes-128-ecb'); --{serverError 42} not enough arguments +SELECT aes_encrypt_mysql('aes-128-ecb', 'text'); --{serverError 42} not enough arguments + +-- Mode +SELECT aes_encrypt_mysql(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT aes_encrypt_mysql('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT aes_encrypt_mysql('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name +SELECT aes_encrypt_mysql('aes-128-gcm', 'text', 'key'); -- {serverError 36} mode is not supported by _mysql-functions + +SELECT encrypt(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT encrypt('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT encrypt('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name + + +-- Key +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short + +SELECT encrypt('aes-128-ecb', 'text'); --{serverError 42} key is missing +SELECT encrypt('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT encrypt('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short +SELECT encrypt('aes-128-ecb', 'text', 'keykeykeykeykeykeykeykeykeykeykeykey'); -- {serverError 36} key is to long + +-- IV +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key', 1011); --{serverError 43} bad IV type 6 +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key', 'iv'); --{serverError 36} IV is too short 4 + +SELECT encrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 1011); --{serverError 43} bad IV type 1 +SELECT encrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iviviviviviviviviviviviviviviviviviviviviv'); --{serverError 36} IV is too long 3 +SELECT encrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iv'); --{serverError 36} IV is too short 2 + +--AAD +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 42} too many arguments + +SELECT encrypt('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type +SELECT encrypt('aes-128-gcm', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type + +----------------------------------------------------------------------------------------- +-- Valid cases +----------------------------------------------------------------------------------------- + +SELECT 'UInt64'; +SELECT hex(aes_encrypt_mysql('aes-128-ecb', 123456789101112, 'keykeykeykeykeykeykeykeykeykeyke')); +SELECT 'Float64'; +SELECT hex(aes_encrypt_mysql('aes-128-ecb', 1234567891011.12, 'keykeykeykeykeykeykeykeykeykeyke')); +SELECT 'Decimal64'; +SELECT hex(aes_encrypt_mysql('aes-128-ecb', toDecimal64(1234567891011.12, 2), 'keykeykeykeykeykeykeykeykeykeyke')); + +----------------------------------------------------------------------------------------- +-- Validate against predefined ciphertext,plaintext,key and IV for MySQL compatibility mode +----------------------------------------------------------------------------------------- +CREATE TABLE encryption_test +( + input String, + key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), + iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85') +) Engine = Memory; + +INSERT INTO encryption_test (input) +VALUES (''), ('text'), ('What Is ClickHouse? ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).'); + +SELECT 'MySQL-specific key folding'; +SELECT 'aes-128-ecb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-ecb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-ecb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cbc' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cbc' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cbc' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cfb1' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cfb1' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cfb1' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cfb8' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cfb8' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cfb8' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cfb128' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cfb128' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cfb128' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-ofb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-ofb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-ofb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'Nullable and LowCardinality'; +WITH CAST(NULL as Nullable(String)) as input, 'aes-256-ofb' as mode SELECT toTypeName(input), hex(aes_encrypt_mysql(mode, input, key,iv)) FROM encryption_test LIMIT 1; +WITH CAST('text' as Nullable(String)) as input, 'aes-256-ofb' as mode SELECT toTypeName(input), hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test LIMIT 1; +WITH CAST('text' as LowCardinality(String)) as input, 'aes-256-ofb' as mode SELECT toTypeName(input), hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test LIMIT 1; + +SELECT 'GCM mode with IV'; +SELECT 'aes-128-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 16), iv)) FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 24), iv)) FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 32), iv)) FROM encryption_test; + +SELECT 'GCM mode with IV and AAD'; +SELECT 'aes-128-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 16), iv, 'AAD')) FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 24), iv, 'AAD')) FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 32), iv, 'AAD')) FROM encryption_test; + +-- based on https://github.com/openssl/openssl/blob/master/demos/evp/aesgcm.c#L20 +WITH + unhex('eebc1f57487f51921c0465665f8ae6d1658bb26de6f8a069a3520293a572078f') as key, + unhex('67ba0510262ae487d737ee6298f77e0c') as tag, + unhex('99aa3e68ed8173a0eed06684') as iv, + unhex('f56e87055bc32d0eeb31b2eacc2bf2a5') as plaintext, + unhex('4d23c3cec334b49bdb370c437fec78de') as aad, + unhex('f7264413a84c0e7cd536867eb9f21736') as ciphertext +SELECT + hex(encrypt('aes-256-gcm', plaintext, key, iv, aad)) as ciphertext_actual, + ciphertext_actual = concat(hex(ciphertext), hex(tag)); diff --git a/tests/ubsan_suppressions.txt b/tests/ubsan_suppressions.txt new file mode 100644 index 00000000000..c29da437a11 --- /dev/null +++ b/tests/ubsan_suppressions.txt @@ -0,0 +1,3 @@ +# Suppress some failures in contrib so that we can enable UBSan in CI. +# Ideally, we should report these upstream. +src:*/contrib/openssl/* \ No newline at end of file From 266b231a3ce300bc1b7b6ea27d69b1084f283596 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 4 Aug 2020 20:06:20 +0200 Subject: [PATCH 027/441] Adding AES encryption tests in TestFlows. --- .../configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 107 + .../configs/clickhouse/config.d/ssl.xml | 17 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../configs/clickhouse/config.xml | 436 ++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../configs/clickhouse/users.xml | 133 + .../configs/clickhouse1/config.d/macros.xml | 8 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../docker-compose/clickhouse-service.yml | 28 + .../docker-compose/docker-compose.yml | 73 + .../docker-compose/mysql-service.yml | 19 + .../docker-compose/zookeeper-service.yml | 18 + tests/testflows/aes_encryption/regression.py | 73 + .../aes_encryption/requirements/__init__.py | 1 + .../requirements/requirements.py | 3663 +++++++++++++++++ .../testflows/aes_encryption/tests/common.py | 162 + .../tests/compatibility/__init__.py | 0 .../tests/compatibility/feature.py | 17 + .../tests/compatibility/insert.py | 414 ++ .../tests/compatibility/mysql/__init__.py | 0 .../compatibility/mysql/database_engine.py | 196 + .../tests/compatibility/mysql/dictionary.py | 251 ++ .../tests/compatibility/mysql/feature.py | 18 + .../tests/compatibility/mysql/table_engine.py | 202 + .../compatibility/mysql/table_function.py | 183 + .../tests/compatibility/select.py | 186 + .../snapshots/insert.py.insert.snapshot | 192 + .../testflows/aes_encryption/tests/decrypt.py | 634 +++ .../aes_encryption/tests/decrypt_mysql.py | 521 +++ .../testflows/aes_encryption/tests/encrypt.py | 408 ++ .../aes_encryption/tests/encrypt_mysql.py | 326 ++ .../snapshots/encrypt.py.encrypt.snapshot | 2700 ++++++++++++ .../encrypt_mysql.py.encrypt_mysql.snapshot | 3060 ++++++++++++++ tests/testflows/regression.py | 1 + 41 files changed, 14176 insertions(+) create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/common.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/users.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml create mode 100644 tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml create mode 100644 tests/testflows/aes_encryption/docker-compose/docker-compose.yml create mode 100644 tests/testflows/aes_encryption/docker-compose/mysql-service.yml create mode 100644 tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/aes_encryption/regression.py create mode 100644 tests/testflows/aes_encryption/requirements/__init__.py create mode 100644 tests/testflows/aes_encryption/requirements/requirements.py create mode 100644 tests/testflows/aes_encryption/tests/common.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/__init__.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/feature.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/insert.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/select.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot create mode 100644 tests/testflows/aes_encryption/tests/decrypt.py create mode 100644 tests/testflows/aes_encryption/tests/decrypt_mysql.py create mode 100644 tests/testflows/aes_encryption/tests/encrypt.py create mode 100644 tests/testflows/aes_encryption/tests/encrypt_mysql.py create mode 100644 tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot create mode 100644 tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot diff --git a/tests/testflows/aes_encryption/configs/clickhouse/common.xml b/tests/testflows/aes_encryption/configs/clickhouse/common.xml new file mode 100644 index 00000000000..df952b28c82 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + 0.0.0.0 + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 +
+
diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..51be2a6e8e3 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,107 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..ca65ffd5e04 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,17 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.xml new file mode 100644 index 00000000000..d34d2c35253 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.xml @@ -0,0 +1,436 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + +
diff --git a/tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/aes_encryption/configs/clickhouse/users.xml b/tests/testflows/aes_encryption/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml b/tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml b/tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml b/tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml b/tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml new file mode 100644 index 00000000000..9787b37abbb --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 3s + timeout: 2s + retries: 40 + start_period: 2s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/aes_encryption/docker-compose/docker-compose.yml b/tests/testflows/aes_encryption/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..04a51ad7ec0 --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/docker-compose.yml @@ -0,0 +1,73 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + mysql1: + extends: + file: mysql-service.yml + service: mysql + hostname: mysql1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/mysql1/database:/var/lib/mysql" + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + mysql1: + condition: service_healthy + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy diff --git a/tests/testflows/aes_encryption/docker-compose/mysql-service.yml b/tests/testflows/aes_encryption/docker-compose/mysql-service.yml new file mode 100644 index 00000000000..6924bccfad5 --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/mysql-service.yml @@ -0,0 +1,19 @@ +version: '2.3' + +services: + mysql: + image: mysql:5.7.30 + restart: always + environment: + MYSQL_DATABASE: 'db' + MYSQL_USER: 'user' + MYSQL_PASSWORD: 'password' + MYSQL_ROOT_PASSWORD: 'password' + expose: + - '3306' + healthcheck: + test: mysql -D db -u user --password=password -e "select 1;" + interval: 3s + timeout: 2s + retries: 40 + start_period: 2s diff --git a/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml b/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml new file mode 100644 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py new file mode 100755 index 00000000000..e50ac0a3f8b --- /dev/null +++ b/tests/testflows/aes_encryption/regression.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from aes_encryption.requirements import * + +xfails = { + # encrypt + "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too short": + [(Fail, "known issue")], + "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too long": + [(Fail, "known issue")], + # encrypt_mysql + "encrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None": + [(Fail, "https://altinity.atlassian.net/browse/CH-190")], + "encrypt_mysql/invalid parameters/iv not valid for mode": + [(Fail, "https://altinity.atlassian.net/browse/CH-190")], + "encrypt_mysql/invalid parameters/no parameters": + [(Fail, "https://altinity.atlassian.net/browse/CH-191")], + # decrypt_mysql + "decrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None:": + [(Fail, "https://altinity.atlassian.net/browse/CH-190")], + # compatibility + "compatibility/insert/encrypt using materialized view/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/insert/decrypt using materialized view/:": + [(Error, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/insert/aes encrypt mysql using materialized view/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/insert/aes decrypt mysql using materialized view/:": + [(Error, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/select/decrypt unique": + [(Fail, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/mysql/:engine/decrypt/mysql_datatype='TEXT'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")], + "compatibility/mysql/:engine/decrypt/mysql_datatype='VARCHAR(100)'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")], + "compatibility/mysql/:engine/encrypt/mysql_datatype='TEXT'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")], + "compatibility/mysql/:engine/encrypt/mysql_datatype='VARCHAR(100)'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")] +} + +@TestFeature +@Name("aes encryption") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS008_AES_Functions("1.0"), + RQ_SRS008_AES_Functions_DifferentModes("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path): + """ClickHouse AES encryption functions regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Feature(run=load("aes_encryption.tests.encrypt", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.decrypt", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.encrypt_mysql", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.decrypt_mysql", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.feature", "feature"), flags=TE) + +if main(): + regression() diff --git a/tests/testflows/aes_encryption/requirements/__init__.py b/tests/testflows/aes_encryption/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/aes_encryption/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/aes_encryption/requirements/requirements.py b/tests/testflows/aes_encryption/requirements/requirements.py new file mode 100644 index 00000000000..bae8b5cc3c1 --- /dev/null +++ b/tests/testflows/aes_encryption/requirements/requirements.py @@ -0,0 +1,3663 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.200731.1222107. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Requirement + +RQ_SRS008_AES_Functions = Requirement( + name='RQ.SRS008.AES.Functions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.Dictionaries', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n' + '[MySQL] servers using [MySQL Dictionary].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_DifferentModes = Requirement( + name='RQ.SRS008.AES.Functions.DifferentModes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n' + 'using explicit function parameters.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( + name='RQ.SRS008.AES.Functions.DataFromMultipleSources', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources\n' + 'in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n' + '[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n' + 'with possibly different encryption schemes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( + name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n' + 'error log, and `query_log` table to prevent leakage of sensitive values.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_InvalidParameters = Requirement( + name='RQ.SRS008.AES.Functions.InvalidParameters', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when parameters are invalid.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return garbage for mismatched keys.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.IV', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.AAD', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error or garbage for mismatched mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Check_Performance = Requirement( + name='RQ.SRS008.AES.Functions.Check.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Performance of [AES] encryption functions SHALL be measured.\n' + ), + link=None + ) + +RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( + name='RQ.SRS008.AES.Function.Check.Performance.BestCase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Performance of [AES] encryption functions SHALL be checked for the best case\n' + 'scenario where there is one key, one initialization vector, and one large stream of data.\n' + ), + link=None + ) + +RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( + name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Performance of [AES] encryption functions SHALL be checked for the worst case\n' + 'where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Check_Compression = Requirement( + name='RQ.SRS008.AES.Functions.Check.Compression', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Effect of [AES] encryption on column compression SHALL be measured.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( + name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n' + 'SHALL be measured.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function = Requirement( + name='RQ.SRS008.AES.Encrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_encrypt` function\n' + '\n' + '```sql\n' + 'aes_encrypt(plaintext, key, mode, [iv, aad])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' + 'the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_encrypt` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' + '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' + 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' + 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' + 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' + 'parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data\n' + 'for block modes that require it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the encrypted value of the data\n' + 'using `String` data type as the result of `aes_encrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' + 'and if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function = Requirement( + name='RQ.SRS008.AES.Decrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_decrypt` function\n' + '\n' + '```sql\n' + 'aes_decrypt(ciphertext, key, mode, [iv, aad])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n' + 'the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_decrypt` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' + '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' + 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' + 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' + 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' + 'parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data\n' + 'for block modes that require it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the decrypted value of the data\n' + 'using `String` data type as the result of `aes_decrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function\n' + 'for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function\n' + 'for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' + 'and if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' + 'or `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function\n' + '\n' + '```sql\n' + 'aes_encrypt_mysql(plaintext, key, mode, [iv])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' + 'the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_encrypt_mysql` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the encrypted value of the data\n' + 'using `String` data type as the result of `aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n' + 'function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n' + 'for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' + '```python\n' + 'def fold_key(key, cipher_key_size):\n' + ' key = list(key) if not isinstance(key, (list, tuple)) else key\n' + '\t folded_key = key[:cipher_key_size]\n' + '\t for i in range(cipher_key_size, len(key)):\n' + '\t\t print(i % cipher_key_size, i)\n' + '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' + '\t return folded_key\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' + 'that is required for the `aes_encrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' + 'its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n' + 'function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' + 'and if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function\n' + '\n' + '```sql\n' + 'aes_decrypt_mysql(ciphertext, key, mode, [iv])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `ciphertext` accepting any data type as\n' + 'the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_decrypt_mysql` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the decrypted value of the data\n' + 'using `String` data type as the result of `aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n' + 'function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n' + 'for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' + '```python\n' + 'def fold_key(key, cipher_key_size):\n' + ' key = list(key) if not isinstance(key, (list, tuple)) else key\n' + '\t folded_key = key[:cipher_key_size]\n' + '\t for i in range(cipher_key_size, len(key)):\n' + '\t\t print(i % cipher_key_size, i)\n' + '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' + '\t return folded_key\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' + 'that is required for the `aes_decrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' + 'its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n' + 'function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' + 'and if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) diff --git a/tests/testflows/aes_encryption/tests/common.py b/tests/testflows/aes_encryption/tests/common.py new file mode 100644 index 00000000000..5ed582563fb --- /dev/null +++ b/tests/testflows/aes_encryption/tests/common.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +modes = [ + # mode, key_len, iv_len, aad + ("'aes-128-ecb'", 16, None, None), + ("'aes-192-ecb'", 24, None, None), + ("'aes-256-ecb'", 32, None, None), + # cbc + ("'aes-128-cbc'", 16, None, None), + ("'aes-192-cbc'", 24, None, None), + ("'aes-256-cbc'", 32, None, None), + ("'aes-128-cbc'", 16, 16, None), + ("'aes-192-cbc'", 24, 16, None), + ("'aes-256-cbc'", 32, 16, None), + # cfb1 + ("'aes-128-cfb1'", 16, None, None), + ("'aes-192-cfb1'", 24, None, None), + ("'aes-256-cfb1'", 32, None, None), + ("'aes-128-cfb1'", 16, 16, None), + ("'aes-192-cfb1'", 24, 16, None), + ("'aes-256-cfb1'", 32, 16, None), + # cfb8 + ("'aes-128-cfb8'", 16, None, None), + ("'aes-192-cfb8'", 24, None, None), + ("'aes-256-cfb8'", 32, None, None), + ("'aes-128-cfb8'", 16, 16, None), + ("'aes-192-cfb8'", 24, 16, None), + ("'aes-256-cfb8'", 32, 16, None), + # cfb128 + ("'aes-128-cfb128'", 16, None, None), + ("'aes-192-cfb128'", 24, None, None), + ("'aes-256-cfb128'", 32, None, None), + ("'aes-128-cfb128'", 16, 16, None), + ("'aes-192-cfb128'", 24, 16, None), + ("'aes-256-cfb128'", 32, 16, None), + # ofb + ("'aes-128-ofb'", 16, None, None), + ("'aes-192-ofb'", 24, None, None), + ("'aes-256-ofb'", 32, None, None), + ("'aes-128-ofb'", 16, 16, None), + ("'aes-192-ofb'", 24, 16, None), + ("'aes-256-ofb'", 32, 16, None), + # gcm + ("'aes-128-gcm'", 16, 12, None), + ("'aes-192-gcm'", 24, 12, None), + ("'aes-256-gcm'", 32, 12, None), + ("'aes-128-gcm'", 16, 12, True), + ("'aes-192-gcm'", 24, 12, True), + ("'aes-256-gcm'", 32, 12, True), + # ctr + ("'aes-128-ctr'", 16, None, None), + ("'aes-192-ctr'", 24, None, None), + ("'aes-256-ctr'", 32, None, None), + ("'aes-128-ctr'", 16, 16, None), + ("'aes-192-ctr'", 24, 16, None), + ("'aes-256-ctr'", 32, 16, None), +] + +mysql_modes = [ + # mode, key_len, iv_len + ("'aes-128-ecb'", 16, None), + ("'aes-128-ecb'", 24, None), + ("'aes-192-ecb'", 24, None), + ("'aes-192-ecb'", 32, None), + ("'aes-256-ecb'", 32, None), + ("'aes-256-ecb'", 64, None), + # cbc + ("'aes-128-cbc'", 16, None), + ("'aes-192-cbc'", 24, None), + ("'aes-256-cbc'", 32, None), + ("'aes-128-cbc'", 16, 16), + ("'aes-128-cbc'", 24, 24), + ("'aes-192-cbc'", 24, 16), + ("'aes-192-cbc'", 32, 32), + ("'aes-256-cbc'", 32, 16), + ("'aes-256-cbc'", 64, 64), + # cfb1 + ("'aes-128-cfb1'", 16, None), + ("'aes-192-cfb1'", 24, None), + ("'aes-256-cfb1'", 32, None), + ("'aes-128-cfb1'", 16, 16), + ("'aes-128-cfb1'", 24, 24), + ("'aes-192-cfb1'", 24, 16), + ("'aes-192-cfb1'", 32, 32), + ("'aes-256-cfb1'", 32, 16), + ("'aes-256-cfb1'", 64, 64), + # cfb8 + ("'aes-128-cfb8'", 16, None), + ("'aes-192-cfb8'", 24, None), + ("'aes-256-cfb8'", 32, None), + ("'aes-128-cfb8'", 16, 16), + ("'aes-128-cfb8'", 24, 24), + ("'aes-192-cfb8'", 24, 16), + ("'aes-192-cfb8'", 32, 32), + ("'aes-256-cfb8'", 32, 16), + ("'aes-256-cfb8'", 64, 64), + # cfb128 + ("'aes-128-cfb128'", 16, None), + ("'aes-192-cfb128'", 24, None), + ("'aes-256-cfb128'", 32, None), + ("'aes-128-cfb128'", 16, 16), + ("'aes-128-cfb128'", 24, 24), + ("'aes-192-cfb128'", 24, 16), + ("'aes-192-cfb128'", 32, 32), + ("'aes-256-cfb128'", 32, 16), + ("'aes-256-cfb128'", 64, 64), + # ofb + ("'aes-128-ofb'", 16, None), + ("'aes-192-ofb'", 24, None), + ("'aes-256-ofb'", 32, None), + ("'aes-128-ofb'", 16, 16), + ("'aes-128-ofb'", 24, 24), + ("'aes-192-ofb'", 24, 16), + ("'aes-192-ofb'", 32, 32), + ("'aes-256-ofb'", 32, 16), + ("'aes-256-ofb'", 64, 64), +] + +plaintexts = [ + ("bytes", "unhex('0')"), + ("emptystring", "''"), + ("utf8string", "'Gãńdåłf_Thê_Gręât'"), + ("utf8fixedstring", "toFixedString('Gãńdåłf_Thê_Gręât', 24)"), + ("String", "'1'"), + ("FixedString", "toFixedString('1', 1)"), + ("UInt8", "toUInt8('1')"), + ("UInt16", "toUInt16('1')"), + ("UInt32", "toUInt32('1')"), + ("UInt64", "toUInt64('1')"), + ("Int8", "toInt8('1')"), + ("Int16", "toInt16('1')"), + ("Int32", "toInt32('1')"), + ("Int64", "toInt64('1')"), + ("Float32", "toFloat32('1')"), + ("Float64", "toFloat64('1')"), + ("Decimal32", "toDecimal32(2, 4)"), + ("Decimal64", "toDecimal64(2, 4)"), + ("Decimal128", "toDecimal128(2, 4)"), + ("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), + ("Date", "toDate('2020-01-01')"), + ("DateTime", "toDateTime('2020-01-01 20:01:02')"), + ("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"), + ("LowCardinality", "toLowCardinality('1')"), + ("Array", "[1,2]"), + #("Tuple", "(1,'a')") - not supported + #("Nullable, "Nullable(X)") - not supported + ("NULL", "toDateOrNull('foo')"), + ("IPv4", "toIPv4('171.225.130.45')"), + ("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"), + ("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"), + ("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')"), +] + +_hex = hex + +def hex(s): + """Convert string to hex. + """ + if isinstance(s, str): + return "".join(['%X' % ord(c) for c in s]) + if isinstance(s, bytes): + return "".join(['%X' % c for c in s]) + return _hex(s) diff --git a/tests/testflows/aes_encryption/tests/compatibility/__init__.py b/tests/testflows/aes_encryption/tests/compatibility/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/aes_encryption/tests/compatibility/feature.py b/tests/testflows/aes_encryption/tests/compatibility/feature.py new file mode 100644 index 00000000000..5ef547e43f4 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/feature.py @@ -0,0 +1,17 @@ +from testflows.core import * + +from aes_encryption.requirements import * + +@TestFeature +@Name("compatibility") +@Requirements( + RQ_SRS008_AES_Functions_DataFromMultipleSources("1.0") +) +def feature(self, node="clickhouse1"): + """Check encryption functions usage compatibility. + """ + self.context.node = self.context.cluster.node(node) + + Feature(run=load("aes_encryption.tests.compatibility.insert", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.select", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.feature", "feature"), flags=TE) \ No newline at end of file diff --git a/tests/testflows/aes_encryption/tests/compatibility/insert.py b/tests/testflows/aes_encryption/tests/compatibility/insert.py new file mode 100644 index 00000000000..6ddcc11b584 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/insert.py @@ -0,0 +1,414 @@ +import os +import textwrap +from contextlib import contextmanager +from importlib.machinery import SourceFileLoader + +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts.helpers import varname +from testflows.asserts import values, error, snapshot + +from aes_encryption.tests.common import modes, mysql_modes + +@contextmanager +def table(name): + node = current().context.node + try: + with Given("table"): + sql = f""" + CREATE TABLE {name} + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """ + with By("dropping table if exists"): + node.query(f"DROP TABLE IF EXISTS {name}") + with And("creating a table"): + node.query(textwrap.dedent(sql)) + yield + finally: + with Finally("I drop the table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {name}") + +@contextmanager +def mv_transform(table, transform): + node = current().context.node + try: + with Given("tables for input transformation"): + with By("creating Null input table"): + sql = f""" + CREATE TABLE {table}_input + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String), + mode String, + key String, + iv String, + aad String + ) + ENGINE=Null() + """ + node.query(textwrap.dedent(sql)) + + with And("creating materialized view table"): + sql = f""" + CREATE MATERIALIZED VIEW {table}_input_mv TO {table} AS + SELECT date, name, {transform} + FROM {table}_input + """ + node.query(textwrap.dedent(sql)) + yield + finally: + with Finally("I drop tables for input transformation", flags=TE): + with By("dropping materialized view table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {table}_input_mv") + + with And("dropping Null input table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {table}_input") + +@TestScenario +def encrypt_using_materialized_view(self): + """Check that we can use `encrypt` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"encrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})" + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) + """) + + with And("I read inserted data back"): + node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_mv_example_{varname(basename(self.name))}")), error() + +@TestScenario +def aes_encrypt_mysql_using_materialized_view(self): + """Check that we can use `aes_encrypt_mysql` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"aes_encrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}) + """) + + with And("I read inserted data back"): + node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_mv_example_{varname(basename(self.name))}")), error() + +@TestScenario +def encrypt_using_input_table_function(self): + """Check that we can use `encrypt` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"encrypt({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})" + + with table("user_data"): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret') + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_input_example_{varname(basename(example.name))}")), error() + +@TestScenario +def aes_encrypt_mysql_using_input_table_function(self): + """Check that we can use `aes_encrypt_mysql` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"aes_encrypt_mysql({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with table("user_data"): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret') + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_input_example_{varname(basename(example.name))}")), error() + +@TestScenario +def decrypt_using_materialized_view(self): + """Check that we can use `decrypt` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"decrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"encrypt_mv_example_{example_name}")) + example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")] + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + with Then("output must match the expected"): + expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'""" + assert r.output == expected, error() + +@TestScenario +def aes_decrypt_mysql_using_materialized_view(self): + """Check that we can use `aes_decrypt_mysql` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"aes_decrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_mv_example_{example_name}")) + example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")] + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}) + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + with Then("output must match the expected"): + expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'""" + assert r.output == expected, error() + +@TestScenario +def decrypt_using_input_table_function(self): + """Check that we can use `decrypt` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"decrypt({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"encrypt_input_example_{example_name}")) + example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")] + + with table("user_data"): + with When("I insert decrypted data"): + node.query(textwrap.dedent(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}') + """)) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret""" + with Then("output must match the expected", description=expected): + assert r.output == expected, error() + +@TestScenario +def aes_decrypt_mysql_using_input_table_function(self): + """Check that we can use `aes_decrypt_mysql` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"aes_decrypt_mysql({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_input_example_{example_name}")) + example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")] + + with table("user_data"): + with When("I insert decrypted data"): + node.query(textwrap.dedent(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}') + """)) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret""" + with Then("output must match the expected", description=expected): + assert r.output == expected, error() + +@TestFeature +@Name("insert") +def feature(self, node="clickhouse1"): + """Check encryption functions when used during data insertion into a table. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py new file mode 100644 index 00000000000..3547dc95ab0 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py @@ -0,0 +1,196 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def table(name, node, mysql_node, secret_type): + """Create a table that can be accessed using MySQL database engine. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I create a database using MySQL database engine"): + sql = f""" + CREATE DATABASE mysql_db + ENGINE = MySQL('{mysql_node.name}:3306', 'db', 'user', 'password') + """ + with When("I drop database if exists"): + node.query(f"DROP DATABASE IF EXISTS mysql_db") + with And("I create database"): + node.query(textwrap.dedent(sql)) + yield + + finally: + with And("I drop the database that is using MySQL database engine", flags=TE): + node.query(f"DROP DATABASE IF EXISTS mysql_db") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a table provided by MySQL database engine that + contains a column encrypted in MySQL stored using specified data type + I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql` + functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL database engine"): + output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_db.user_data") + + with And("I read decrypted data using MySQL database engine"): + output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip() + + with Then("output should match the original plain text"): + assert output == hex("secret"), error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def encrypt(self, mysql_datatype): + """Check that when using a table provided by MySQL database engine that + we can encrypt data during insert using the `aes_encrypt_mysql` function + and decrypt it in MySQL. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["encrypt", "aes_encrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "encrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with When("I insert encrypted data into a table provided by MySQL database engine"): + node.query(textwrap.dedent(f""" + INSERT INTO + mysql_db.user_data + SELECT + id, date, name, {example_transform} + FROM + input('id Int32, date Date, name String, secret String') + FORMAT Values (1, '2020-01-01', 'user0', 'secret') + """)) + + with And("I read decrypted data using MySQL database engine"): + output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip() + + with Then("decrypted data from MySQL database engine should should match the original plain text"): + assert output == hex("secret"), error() + + with And("I read raw data using MySQL database engine to get expected raw data"): + expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_db.user_data").output.strip() + + with And("I read raw encrypted data in MySQL"): + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + + with Then("check that raw encryted data in MySQL matches the expected"): + assert expected_raw_data in output, error() + + with And("I decrypt data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; + """ + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + + with Then("decryted data in MySQL should match the original plain text"): + assert hex("secret") in output, error() + +@TestFeature +@Name("database engine") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL database engine]. + + [MySQL database engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/ + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py new file mode 100644 index 00000000000..66b9e3acbf8 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py @@ -0,0 +1,251 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def dictionary(name, node, mysql_node, secret_type): + """Create a table in MySQL and use it a source for a dictionary. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("dictionary that uses MySQL table as the external source"): + with When("I drop the dictionary if exists"): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + with And("I create the dictionary"): + sql = f""" + CREATE DICTIONARY dict_{name} + ( + id Int32, + date Date, + name String, + secret String + ) + PRIMARY KEY id + SOURCE(MYSQL( + USER 'user' + PASSWORD 'password' + DB 'db' + TABLE '{name}' + REPLICA(PRIORITY 1 HOST '{mysql_node.name}' PORT 3306) + )) + LAYOUT(HASHED()) + LIFETIME(0) + """ + node.query(textwrap.dedent(sql)) + + yield f"dict_{name}" + + finally: + with Finally("I drop the dictionary", flags=TE): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@contextmanager +def parameters_dictionary(name, node, mysql_node): + """Create a table in MySQL and use it a source for a dictionary + that stores parameters for the encryption functions. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + `id` INT NOT NULL AUTO_INCREMENT, + `name` VARCHAR(100), + `mode` VARCHAR(100), + `key` BLOB, + `iv` BLOB, + `text` BLOB, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("dictionary that uses MySQL table as the external source"): + with When("I drop the dictionary if exists"): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + with And("I create the dictionary"): + sql = f""" + CREATE DICTIONARY dict_{name} + ( + id Int32, + name String, + mode String, + key String, + iv String, + text String + ) + PRIMARY KEY id + SOURCE(MYSQL( + USER 'user' + PASSWORD 'password' + DB 'db' + TABLE '{name}' + REPLICA(PRIORITY 1 HOST '{mysql_node.name}' PORT 3306) + )) + LAYOUT(HASHED()) + LIFETIME(0) + """ + node.query(textwrap.dedent(sql)) + + yield f"dict_{name}" + + finally: + with Finally("I drop the dictionary", flags=TE): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestScenario +def parameter_values(self): + """Check that we can use a dictionary that uses MySQL table as a source + can be used as a parameters store for the `encrypt`, `decrypt`, and + `aes_encrypt_mysql`, `aes_decrypt_mysql` functions. + """ + node = self.context.node + mysql_node = self.context.mysql_node + mode = "'aes-128-cbc'" + key = f"'{'1' * 16}'" + iv = f"'{'2' * 16}'" + plaintext = "'secret'" + + for encrypt, decrypt in [ + ("encrypt", "decrypt"), + ("aes_encrypt_mysql", "aes_decrypt_mysql") + ]: + with Example(f"{encrypt} and {decrypt}", description=f"Check using dictionary for parameters of {encrypt} and {decrypt} functions."): + with parameters_dictionary("parameters_data", node, mysql_node) as dict_name: + with When("I insert parameters values in MySQL"): + sql = f""" + INSERT INTO parameters_data VALUES (1, 'user0', {mode}, {key}, {iv}, {plaintext}); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I use dictionary values as parameters"): + sql = f""" + SELECT {decrypt}( + dictGet('default.{dict_name}', 'mode', toUInt64(1)), + {encrypt}( + dictGet('default.{dict_name}', 'mode', toUInt64(1)), + dictGet('default.{dict_name}', 'text', toUInt64(1)), + dictGet('default.{dict_name}', 'key', toUInt64(1)), + dictGet('default.{dict_name}', 'iv', toUInt64(1)) + ), + dictGet('default.{dict_name}', 'key', toUInt64(1)), + dictGet('default.{dict_name}', 'iv', toUInt64(1)) + ) + """ + output = node.query(textwrap.dedent(sql)).output.strip() + + with Then("output should match the plain text"): + assert f"'{output}'" == plaintext, error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a dictionary that uses MySQL table as a source and + contains a data encrypted in MySQL and stored using specified data type + that we can decrypt data from the dictionary using + the `aes_decrypt_mysql` or `decrypt` functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with dictionary("user_data", node, mysql_node, mysql_datatype) as dict_name: + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL dictionary"): + output = node.query(f"SELECT hex(dictGet('default.{dict_name}', 'secret', toUInt64(1))) AS secret") + + with And("I read decrypted data using MySQL dictionary"): + output = node.query(textwrap.dedent(f""" + SELECT hex( + {func}( + {example_mode}, + dictGet('default.{dict_name}', 'secret', toUInt64(1)), + {example_key}{(", " + example_iv) if example_iv else ""} + ) + ) + """)).output.strip() + + with Then("output should match the original plain text"): + assert output == hex("secret"), error() + +@TestFeature +@Name("dictionary") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_Dictionaries("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL dictionary]. + + [MySQL dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py new file mode 100644 index 00000000000..5c6338bfdff --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py @@ -0,0 +1,18 @@ +from testflows.core import * + +from aes_encryption.requirements import * + +@TestFeature +@Name("mysql") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_MySQL("1.0") +) +def feature(self, node="clickhouse1"): + """Check encryption functions usage compatibility with MySQL. + """ + self.context.node = self.context.cluster.node(node) + + Feature(run=load("aes_encryption.tests.compatibility.mysql.table_engine", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.database_engine", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.table_function", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.dictionary", "feature"), flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py new file mode 100644 index 00000000000..5a5c7d9d58a --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py @@ -0,0 +1,202 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def table(name, node, mysql_node, secret_type): + """Create a table that can be accessed using MySQL table engine. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I create a table using MySQL table engine"): + sql = f""" + CREATE TABLE mysql_{name} + ( + id Nullable(Int32), + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = MySQL('{mysql_node.name}:3306', 'db', '{name}', 'user', 'password') + """ + with When("I drop table if exists"): + node.query(f"DROP TABLE IF EXISTS mysql_{name}") + with And("I create table"): + node.query(textwrap.dedent(sql)) + yield + + finally: + with And("I drop a table using MySQL table engine", flags=TE): + node.query(f"DROP TABLE IF EXISTS mysql_{name}") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a table with MySQL table engine that + contains a column encrypted in MySQL stored using specified data type + I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql` + functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL table engine"): + output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_user_data") + + with And("I read decrypted data via MySQL table engine"): + output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip() + + with Then("the output should match the original plain text"): + assert output == hex("secret"), error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def encrypt(self, mysql_datatype): + """Check that when using a table with MySQL table engine that + we can encrypt data during insert using the `encrypt` and `aes_encrypt_mysql` + functions and decrypt it in MySQL. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["encrypt", "aes_encrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "encrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with When("I insert encrypted data into MySQL table engine"): + node.query(textwrap.dedent(f""" + INSERT INTO + mysql_user_data + SELECT + id, date, name, {example_transform} + FROM + input('id Nullable(Int32), date Date, name String, secret String') + FORMAT Values (null, '2020-01-01', 'user0', 'secret') + """)) + + with And("I read decrypted data via MySQL table engine"): + output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip() + + with Then("decrypted data from MySQL table engine should should match the original plain text"): + assert output == hex("secret"), error() + + with And("I read raw data using MySQL table engine to get expected raw data"): + expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_user_data").output.strip() + + with And("I read raw encrypted data in MySQL"): + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + + with Then("check that raw encryted data in MySQL matches the expected"): + assert expected_raw_data in output, error() + + with And("I decrypt data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; + """ + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + + with Then("decryted data in MySQL should match the original plain text"): + assert hex("secret") in output, error() + +@TestFeature +@Name("table engine") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL table engine]. + + [MySQL table engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/ + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py new file mode 100644 index 00000000000..cd3487c5c74 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py @@ -0,0 +1,183 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def table(name, node, mysql_node, secret_type): + """Create a table that can be accessed using MySQL table function. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + yield f"mysql('{mysql_node.name}:3306', 'db', 'user_data', 'user', 'password')" + + finally: + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a table accessed through MySQL table function that + contains a column encrypted in MySQL stored using specified data type + I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql` + functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype) as table_function: + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL table function"): + output = node.query(f"SELECT id, date, name, hex(secret) AS secret FROM {table_function}") + + with And("I read decrypted data using MySQL table function"): + output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip() + + with Then("output should match the original plain text"): + assert output == hex("secret"), error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def encrypt(self, mysql_datatype): + """Check that when using a table accessed through MySQL table function that + we can encrypt data during insert using the `aes_encrypt_mysql` function + and decrypt it in MySQL. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["encrypt", "aes_encrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "encrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype) as table_function: + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with When("I insert encrypted data into a table provided by MySQL database engine"): + node.query(textwrap.dedent(f""" + INSERT INTO TABLE FUNCTION + {table_function} + SELECT + id, date, name, {example_transform} + FROM + input('id Int32, date Date, name String, secret String') + FORMAT Values (1, '2020-01-01', 'user0', 'secret') + """)) + + with And("I read decrypted data using MySQL database engine"): + output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip() + + with Then("decrypted data from MySQL database engine should should match the original plain text"): + assert output == hex("secret"), error() + + with And("I read raw data using MySQL database engine to get expected raw data"): + expected_raw_data = node.query(f"SELECT hex(secret) AS secret FROM {table_function}").output.strip() + + with And("I read raw encrypted data in MySQL"): + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + + with Then("check that raw encryted data in MySQL matches the expected"): + assert expected_raw_data in output, error() + + with And("I decrypt data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; + """ + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + + with Then("decryted data in MySQL should match the original plain text"): + assert hex("secret") in output, error() + +@TestFeature +@Name("table function") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL table function]. + + [MySQL table function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/ + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) \ No newline at end of file diff --git a/tests/testflows/aes_encryption/tests/compatibility/select.py b/tests/testflows/aes_encryption/tests/compatibility/select.py new file mode 100644 index 00000000000..f81920c65d3 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/select.py @@ -0,0 +1,186 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts.helpers import varname +from testflows.asserts import values, error, snapshot + +from aes_encryption.tests.common import modes, mysql_modes + +@contextmanager +def table(name, sql): + node = current().context.node + try: + with Given("table"): + + with By("dropping table if exists"): + node.query(f"DROP TABLE IF EXISTS {name}") + with And("creating a table"): + node.query(textwrap.dedent(sql.format(name=name))) + yield + finally: + with Finally("I drop the table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {name}") + +@TestScenario +def decrypt(self): + """Check decrypting column when reading data from a table. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example: + with table("user_table", """ + CREATE TABLE {name} + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """): + + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + + with When("I insert encrypted data"): + encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip() + node.query(textwrap.dedent(f""" + INSERT INTO user_table + (date, name, secret) + VALUES + ('2020-01-01', 'user0', unhex('{encrypted_secret}')) + """)) + + with And("I decrypt data during query"): + output = node.query(f"""SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table FORMAT JSONEachRow""").output.strip() + + with Then("I should get back the original plain text"): + assert output == '{"name":"user0","secret":"secret"}', error() + +@TestScenario +def decrypt_multiple(self, count=1000): + """Check decrypting column when reading multiple entries + encrypted with the same parameters for the same user + from a table. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example: + with table("user_table", """ + CREATE TABLE {name} + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """): + + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + + with When("I insert encrypted data"): + encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip() + values = [f"('2020-01-01', 'user0', unhex('{encrypted_secret}'))"] * count + node.query( + "INSERT INTO user_table\n" + " (date, name, secret)\n" + f"VALUES {', '.join(values)}") + + with And("I decrypt data", description="using a subquery and get the number of entries that match the plaintext"): + output = node.query(f"""SELECT count() AS count FROM (SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table) WHERE secret = 'secret' FORMAT JSONEachRow""").output.strip() + + with Then("I should get back the expected result", description=f"{count}"): + assert output == f'{{"count":"{count}"}}', error() + +@TestScenario +def decrypt_unique(self): + """Check decrypting column when reading multiple entries + encrypted with the different parameters for each user + from a table. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + with table("user_table", """ + CREATE TABLE {name} + ( + id UInt64, + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """): + + user_modes = [] + user_keys = [] + + with When("I get encrypted data"): + user_id = 0 + values = [] + + for mode, key_len, iv_len, aad_len in modes: + if "gcm" in mode: + continue + user_modes.append(mode) + user_keys.append(f"'{key[:key_len]}'") + + with When(f"I get encrypted data for user {user_id}"): + encrypted_secret = node.query( + f"""SELECT hex(encrypt({user_modes[-1]}, 'secret', {user_keys[-1]}))""" + ).output.strip() + values.append(f"({user_id}, '2020-01-01', 'user{user_id}', unhex('{encrypted_secret}'))") + + user_id += 1 + + with And("I insert encrypted data for all users"): + node.query( + "INSERT INTO user_table\n" + " (id, date, name, secret)\n" + f"VALUES {', '.join(values)}") + + with And("I read decrypted data for all users"): + output = node.query(textwrap.dedent(f""" + SELECT + count() AS count + FROM + ( + SELECT + [{",".join(user_modes)}] AS modes, + [{",".join(user_keys)}] AS keys, + name, + decrypt(modes[id], secret, keys[id]) AS secret + FROM user_table + ) + WHERE + secret = 'secret' + FORMAT JSONEachRow + """)).output.strip() + + with Then("I should get back the expected result", description=f"{count}"): + assert output == f'{{"count":"{count}"}}', error() + +@TestFeature +@Name("select") +def feature(self, node="clickhouse1"): + """Check encryption functions when used during table querying. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot b/tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot new file mode 100644 index 00000000000..93d915bb9d6 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot @@ -0,0 +1,192 @@ +aes_encrypt_mysql_input_example_mode_aes_128_ecb_key_16_iv_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ecb_key_24_iv_None = r"""'2020-01-01\tuser0\tB418FF12BCBF9E42FA7C19D6EE26BF0B\n2020-01-02\tuser1\t3147A3FEE47DF418D1D75CBC1BC14DE6\n2020-01-03\tuser2\tAECEFD40C6632A0FC033D040E44CCBCC'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ecb_key_24_iv_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ecb_key_32_iv_None = r"""'2020-01-01\tuser0\t044E715357AF77234FD95359666CAFF3\n2020-01-02\tuser1\tB633EF852CE85B4C97827401FD9B606B\n2020-01-03\tuser2\t2AFF7052C748E4BC3BDA8460AFD5A21D'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ecb_key_32_iv_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ecb_key_64_iv_None = r"""'2020-01-01\tuser0\tBFFEC9DF7285A3EC799C941E1450839C\n2020-01-02\tuser1\t3EA0ECBD06326D227A7B9519B1A2955D\n2020-01-03\tuser2\t1478C57DD49523ABDB83A0917F0EDA60'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_16_iv_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_24_iv_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_32_iv_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_16_iv_16 = r"""'2020-01-01\tuser0\tFC93C1D5E5E3B054C1F3A5692AAC0A61\n2020-01-02\tuser1\tD6DBC76ABCB14B7C6D93F1A5FCA66B9C\n2020-01-03\tuser2\tD4F4158A650D01EB505CC72EFE455486'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_24_iv_24 = r"""'2020-01-01\tuser0\t26CEE6B6EBDDE1BF887FDEB75F28FB52\n2020-01-02\tuser1\tF9EC1A75BEEFF70B4DEB39AAD075CEFF\n2020-01-03\tuser2\t3FF84AB3BD40FAEEF70F06BCF6AF9C42'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_24_iv_16 = r"""'2020-01-01\tuser0\t0E3BAF7F4E0BFCFFAE2589B67F71E277\n2020-01-02\tuser1\t2581CCEE9ABE5770480901D65B3D9222\n2020-01-03\tuser2\tED9F3BD8DB12FDF9F2462FFA572361E7'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_32_iv_32 = r"""'2020-01-01\tuser0\t07371B5DE2E378EE08A3A8B6B9FEAD13\n2020-01-02\tuser1\t3C0BF5D187421ECFFD3E00474A154452\n2020-01-03\tuser2\t05B253FA783D78D864AF7C4D5E6A492D'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_32_iv_16 = r"""'2020-01-01\tuser0\t72AC7BA6F283EA94A3C33C4D3E51C7D3\n2020-01-02\tuser1\tDACBBE79062F1C721A01CEEE3E85524F\n2020-01-03\tuser2\tFF5A09D19E5EB2ADD94581308588E44A'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_64_iv_64 = r"""'2020-01-01\tuser0\t573924F0BB4AA1780D45DB6451F123D6\n2020-01-02\tuser1\t007A54AA7ADE8EF844D28936486D75BC\n2020-01-03\tuser2\tAA7249B514398FE1EE827C44402BCE57'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_16_iv_None = r"""'2020-01-01\tuser0\t750BE8662F57A095EC0E610C\n2020-01-02\tuser1\t750BE8662E444A6284C0FC72\n2020-01-03\tuser2\t750BE8662C000B61CDCF1C94'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_24_iv_None = r"""'2020-01-01\tuser0\t5DCC67A043EB776D8B7F5B70\n2020-01-02\tuser1\t5DCC67A042B46DFCC10EFD66\n2020-01-03\tuser2\t5DCC67A040243A8C1346D2DD'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_32_iv_None = r"""'2020-01-01\tuser0\tFAAC1A7D2CE844F8DEB4C44E\n2020-01-02\tuser1\tFAAC1A7D2DF85A43828C0FF8\n2020-01-03\tuser2\tFAAC1A7D2FC7582CCEFCF330'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_16_iv_16 = r"""'2020-01-01\tuser0\t7670A865D13B1B65AD46F8ED\n2020-01-02\tuser1\t7670A865D046007A1E218286\n2020-01-03\tuser2\t7670A865D2E5B091492ECCFB'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_24_iv_24 = r"""'2020-01-01\tuser0\t51EADDE82195C31118D0C171\n2020-01-02\tuser1\t51EADDE82009A46518270271\n2020-01-03\tuser2\t51EADDE8235CB38F95766481'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_24_iv_16 = r"""'2020-01-01\tuser0\t7F38C051539074C0A635C937\n2020-01-02\tuser1\t7F38C051520A30DFACBE9564\n2020-01-03\tuser2\t7F38C051500DA29FF0E7B799'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_32_iv_32 = r"""'2020-01-01\tuser0\t2067186DB91666DE730D0708\n2020-01-02\tuser1\t2067186DB83E2E8B0019F839\n2020-01-03\tuser2\t2067186DBB540332BFC84955'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_32_iv_16 = r"""'2020-01-01\tuser0\t0A216A58A5C0A33215E8E722\n2020-01-02\tuser1\t0A216A58A4E94067ABF030B6\n2020-01-03\tuser2\t0A216A58A6822CAB0318C632'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_64_iv_64 = r"""'2020-01-01\tuser0\t81BD636E1BF4CA02399943E3\n2020-01-02\tuser1\t81BD636E1A93D5D6DD9DCD8D\n2020-01-03\tuser2\t81BD636E18F15168D19C8117'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_16_iv_None = r"""'2020-01-01\tuser0\t650D96B9698D20DB12E2E437\n2020-01-02\tuser1\t650D96B968F00D16ABF2852E\n2020-01-03\tuser2\t650D96B96B8141F425E60D6B'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_24_iv_None = r"""'2020-01-01\tuser0\t72C4724B2F528724A12041C0\n2020-01-02\tuser1\t72C4724B2EF3C6A6FF9E09A9\n2020-01-03\tuser2\t72C4724B2D6EAB1D47709E15'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FD6C94961765ED204F2BCA\n2020-01-02\tuser1\tC5FD6C9497AB1C1AF1DE671C\n2020-01-03\tuser2\tC5FD6C949491F4A3EA5069B3'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_16_iv_16 = r"""'2020-01-01\tuser0\t471D217E9CA3593FFEC955C8\n2020-01-02\tuser1\t471D217E9D7F484D85F81F19\n2020-01-03\tuser2\t471D217E9EBBFD2EA9841008'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_24_iv_24 = r"""'2020-01-01\tuser0\t2EE6147B830481BE36CBE350\n2020-01-02\tuser1\t2EE6147B82DE8F3197AF17A6\n2020-01-03\tuser2\t2EE6147B81FF826E798A0355'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_24_iv_16 = r"""'2020-01-01\tuser0\t1D98EFFAEB9907457BD3FCB2\n2020-01-02\tuser1\t1D98EFFAEA2D930825C6AE22\n2020-01-03\tuser2\t1D98EFFAE92C1D018438B98B'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_32_iv_32 = r"""'2020-01-01\tuser0\t4410165F7DCFDDBB1B15573F\n2020-01-02\tuser1\t4410165F7CFE6A0D2FD5CA9C\n2020-01-03\tuser2\t4410165F7FE8E0C081B3FB7B'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C07B443BB7D7D60E9999C1D\n2020-01-02\tuser1\t1C07B443BA9674A3F68FF3FE\n2020-01-03\tuser2\t1C07B443B95F4B68161A616F'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_64_iv_64 = r"""'2020-01-01\tuser0\tA6D2368A5F177157D73FBD9D\n2020-01-02\tuser1\tA6D2368A5E695ADF99475359\n2020-01-03\tuser2\tA6D2368A5DB96AFD43311124'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_16_iv_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_24_iv_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_16_iv_16 = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_24_iv_24 = r"""'2020-01-01\tuser0\t2E046787D9EFFED25D69C908\n2020-01-02\tuser1\t2E046787D8EFFED25D69C908\n2020-01-03\tuser2\t2E046787DBEFFED25D69C908'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_24_iv_16 = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_32_iv_32 = r"""'2020-01-01\tuser0\t44D3EB069FF443A121590842\n2020-01-02\tuser1\t44D3EB069EF443A121590842\n2020-01-03\tuser2\t44D3EB069DF443A121590842'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_64_iv_64 = r"""'2020-01-01\tuser0\tA69DAA2E8B265618D25D5FE4\n2020-01-02\tuser1\tA69DAA2E8A265618D25D5FE4\n2020-01-03\tuser2\tA69DAA2E89265618D25D5FE4'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_16_iv_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_24_iv_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_16_iv_16 = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_24_iv_24 = r"""'2020-01-01\tuser0\t2E046787D9EFFED25D69C908\n2020-01-02\tuser1\t2E046787D8EFFED25D69C908\n2020-01-03\tuser2\t2E046787DBEFFED25D69C908'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_24_iv_16 = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_32_iv_32 = r"""'2020-01-01\tuser0\t44D3EB069FF443A121590842\n2020-01-02\tuser1\t44D3EB069EF443A121590842\n2020-01-03\tuser2\t44D3EB069DF443A121590842'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_64_iv_64 = r"""'2020-01-01\tuser0\tA69DAA2E8B265618D25D5FE4\n2020-01-02\tuser1\tA69DAA2E8A265618D25D5FE4\n2020-01-03\tuser2\tA69DAA2E89265618D25D5FE4'""" + +encrypt_input_example_mode_aes_128_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +encrypt_input_example_mode_aes_192_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +encrypt_input_example_mode_aes_256_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +encrypt_input_example_mode_aes_128_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +encrypt_input_example_mode_aes_192_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +encrypt_input_example_mode_aes_256_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +encrypt_input_example_mode_aes_128_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\tFC93C1D5E5E3B054C1F3A5692AAC0A61\n2020-01-02\tuser1\tD6DBC76ABCB14B7C6D93F1A5FCA66B9C\n2020-01-03\tuser2\tD4F4158A650D01EB505CC72EFE455486'""" + +encrypt_input_example_mode_aes_192_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\t0E3BAF7F4E0BFCFFAE2589B67F71E277\n2020-01-02\tuser1\t2581CCEE9ABE5770480901D65B3D9222\n2020-01-03\tuser2\tED9F3BD8DB12FDF9F2462FFA572361E7'""" + +encrypt_input_example_mode_aes_256_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\t72AC7BA6F283EA94A3C33C4D3E51C7D3\n2020-01-02\tuser1\tDACBBE79062F1C721A01CEEE3E85524F\n2020-01-03\tuser2\tFF5A09D19E5EB2ADD94581308588E44A'""" + +encrypt_input_example_mode_aes_128_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\t750BE8662F57A095EC0E610C\n2020-01-02\tuser1\t750BE8662E444A6284C0FC72\n2020-01-03\tuser2\t750BE8662C000B61CDCF1C94'""" + +encrypt_input_example_mode_aes_192_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\t5DCC67A043EB776D8B7F5B70\n2020-01-02\tuser1\t5DCC67A042B46DFCC10EFD66\n2020-01-03\tuser2\t5DCC67A040243A8C1346D2DD'""" + +encrypt_input_example_mode_aes_256_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\tFAAC1A7D2CE844F8DEB4C44E\n2020-01-02\tuser1\tFAAC1A7D2DF85A43828C0FF8\n2020-01-03\tuser2\tFAAC1A7D2FC7582CCEFCF330'""" + +encrypt_input_example_mode_aes_128_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t7670A865D13B1B65AD46F8ED\n2020-01-02\tuser1\t7670A865D046007A1E218286\n2020-01-03\tuser2\t7670A865D2E5B091492ECCFB'""" + +encrypt_input_example_mode_aes_192_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t7F38C051539074C0A635C937\n2020-01-02\tuser1\t7F38C051520A30DFACBE9564\n2020-01-03\tuser2\t7F38C051500DA29FF0E7B799'""" + +encrypt_input_example_mode_aes_256_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t0A216A58A5C0A33215E8E722\n2020-01-02\tuser1\t0A216A58A4E94067ABF030B6\n2020-01-03\tuser2\t0A216A58A6822CAB0318C632'""" + +encrypt_input_example_mode_aes_128_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\t650D96B9698D20DB12E2E437\n2020-01-02\tuser1\t650D96B968F00D16ABF2852E\n2020-01-03\tuser2\t650D96B96B8141F425E60D6B'""" + +encrypt_input_example_mode_aes_192_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C4724B2F528724A12041C0\n2020-01-02\tuser1\t72C4724B2EF3C6A6FF9E09A9\n2020-01-03\tuser2\t72C4724B2D6EAB1D47709E15'""" + +encrypt_input_example_mode_aes_256_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FD6C94961765ED204F2BCA\n2020-01-02\tuser1\tC5FD6C9497AB1C1AF1DE671C\n2020-01-03\tuser2\tC5FD6C949491F4A3EA5069B3'""" + +encrypt_input_example_mode_aes_128_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t471D217E9CA3593FFEC955C8\n2020-01-02\tuser1\t471D217E9D7F484D85F81F19\n2020-01-03\tuser2\t471D217E9EBBFD2EA9841008'""" + +encrypt_input_example_mode_aes_192_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t1D98EFFAEB9907457BD3FCB2\n2020-01-02\tuser1\t1D98EFFAEA2D930825C6AE22\n2020-01-03\tuser2\t1D98EFFAE92C1D018438B98B'""" + +encrypt_input_example_mode_aes_256_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C07B443BB7D7D60E9999C1D\n2020-01-02\tuser1\t1C07B443BA9674A3F68FF3FE\n2020-01-03\tuser2\t1C07B443B95F4B68161A616F'""" + +encrypt_input_example_mode_aes_128_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +encrypt_input_example_mode_aes_192_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +encrypt_input_example_mode_aes_256_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +encrypt_input_example_mode_aes_128_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +encrypt_input_example_mode_aes_192_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +encrypt_input_example_mode_aes_256_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +encrypt_input_example_mode_aes_128_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +encrypt_input_example_mode_aes_192_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +encrypt_input_example_mode_aes_256_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +encrypt_input_example_mode_aes_128_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +encrypt_input_example_mode_aes_192_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +encrypt_input_example_mode_aes_256_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +encrypt_input_example_mode_aes_128_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t98E5A430C4A01C4429B0F37A4B3CDBC2BDB491651A36D7F904E231E0\n2020-01-02\tuser1\t98E5A430C5A01C4429B0F37A6E108322C2863C1ABF9BC7098CD369DB\n2020-01-03\tuser2\t98E5A430C6A01C4429B0F37A01646A0243D1CB9A516CF61814808196'""" + +encrypt_input_example_mode_aes_192_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t3F89C3B657596C86202B59F4350807B364DA1E94682EAB679617575D\n2020-01-02\tuser1\t3F89C3B656596C86202B59F4FA03602ED37788B312FDE2AFDBB7F097\n2020-01-03\tuser2\t3F89C3B655596C86202B59F4691EC8880B8132DA9D8838F70D5618C8'""" + +encrypt_input_example_mode_aes_256_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t23B80948CCDB54DC6D0B62F215132A07B30BA6F15593B4F946726B11\n2020-01-02\tuser1\t23B80948CDDB54DC6D0B62F2A01C1BAE07B8D6B26F60116040CDDB55\n2020-01-03\tuser2\t23B80948CEDB54DC6D0B62F2BD0D4954DA6D46772074FFCB4B0D0B98'""" + +encrypt_input_example_mode_aes_128_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t98E5A430C4A01C4429B0F37AF9758E0EA4B44A50A7F964C8E51A913C\n2020-01-02\tuser1\t98E5A430C5A01C4429B0F37ADC59D6EEDB86E72F025474386D2BC907\n2020-01-03\tuser2\t98E5A430C6A01C4429B0F37AB32D3FCE5AD110AFECA34529F578214A'""" + +encrypt_input_example_mode_aes_192_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t3F89C3B657596C86202B59F4B6C662DFF6347EF3B46C170A2F80E946\n2020-01-02\tuser1\t3F89C3B656596C86202B59F479CD05424199E8D4CEBF5EC262204E8C\n2020-01-03\tuser2\t3F89C3B655596C86202B59F4EAD0ADE4996F52BD41CA849AB4C1A6D3'""" + +encrypt_input_example_mode_aes_256_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t23B80948CCDB54DC6D0B62F28787710BBF3F9A594C387B9F7CA2372B\n2020-01-02\tuser1\t23B80948CDDB54DC6D0B62F2328840A20B8CEA1A76CBDE067A1D876F\n2020-01-03\tuser2\t23B80948CEDB54DC6D0B62F22F991258D6597ADF39DF30AD71DD57A2'""" + +encrypt_input_example_mode_aes_128_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +encrypt_input_example_mode_aes_192_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +encrypt_input_example_mode_aes_256_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +encrypt_input_example_mode_aes_128_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +encrypt_input_example_mode_aes_192_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +encrypt_input_example_mode_aes_256_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + diff --git a/tests/testflows/aes_encryption/tests/decrypt.py b/tests/testflows/aes_encryption/tests/decrypt.py new file mode 100644 index 00000000000..faba5363f2f --- /dev/null +++ b/tests/testflows/aes_encryption/tests/decrypt.py @@ -0,0 +1,634 @@ +# -*- coding: utf-8 -*- +import os +from importlib.machinery import SourceFileLoader + +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts.helpers import varname +from testflows.asserts import error + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def decrypt(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When, cast=None, endcast=None, compare=None, no_checks=False): + """Execute `decrypt` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if ciphertext is not None: + params.append(ciphertext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + if aad is not None: + params.append(aad) + + sql = f"decrypt(" + ", ".join(params) + ")" + if cast: + sql = f"{cast}({sql}){endcast or ''}" + if compare: + sql = f"{compare} = {sql}" + sql = f"SELECT {sql}" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"), +) +def invalid_ciphertext(self): + """Check that `decrypt` function does not crash when invalid + `ciphertext` is passed as the first parameter. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + invalid_ciphertexts = plaintexts + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}"""): + d_iv = None if not iv_len else f"'{iv[:iv_len]}'" + d_aad = None if not aad_len else f"'{aad}'" + + for datatype, ciphertext in invalid_ciphertexts: + if datatype in ["NULL"]: + continue + with When(f"invalid ciphertext={ciphertext}"): + if "cfb" in mode or "ofb" in mode or "ctr" in mode: + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, cast="hex") + else: + with When("I execute decrypt function"): + r = decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, no_checks=True, step=By) + with Then("exitcode is not zero"): + assert r.exitcode in [198, 36] + with And("exception is present in the output"): + assert "DB::Exception:" in r.output + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `decrypt` function returns an error when + we call it with invalid parameters. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + + with Example("no parameters"): + decrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function decrypt provided 0, expected 3 to 5") + + with Example("missing key and mode"): + decrypt(ciphertext=ciphertext, exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 1") + + with Example("missing mode"): + decrypt(ciphertext=ciphertext, key="'123'", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 2") + + with Example("bad key type - UInt8"): + decrypt(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("bad aad type - UInt8"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode") + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # ECB + ("'aes-128-ecb'", 16, None, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), + # CTR + ("'aes-128-ctr'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ctr'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ctr'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the mode. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When("iv is too short"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + if aad is None: + with When("aad is specified but not needed"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode") + + else: + with When("iv is specified but not needed"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'"))) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # GCM + ("'aes-128-gcm'", 16, 8, "'hello there aad'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-128-gcm'", 16, None, "'hello there aad'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-gcm'", 24, 8, "''", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-gcm'", 24, None, "''", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-gcm'", 32, 8, "'a'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-gcm'", 32, None, "'a'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the GCM mode. + """ + ciphertext = "'hello there'" + plaintext = "hello there" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When(f"iv is too short"): + ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=198, message="DB::Exception:") + else: + with When("iv is not specified"): + ciphertext = "unhex('1CD4EC93A4B0C687926E8F8C2AA3B4CE1943D006DAE3A774CB1AE5')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to set custom IV length to 0") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"), + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length("1.0") +) +def aad_parameter_types_and_length(self): + """Check that `decrypt` function accepts `aad` parameter as the fifth argument + of either `String` or `FixedString` types and that the length is not limited. + """ + plaintext = "hello there" + iv = "'012345678912'" + mode = "'aes-128-gcm'" + key = "'0123456789123456'" + + with When("aad is specified using String type"): + ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'aad'", message=plaintext) + + with When("aad is specified using String with UTF8 characters"): + ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message=plaintext) + + with When("aad is specified using FixedString type"): + ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message=plaintext) + + with When("aad is specified using FixedString with UTF8 characters"): + ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message=plaintext) + + with When("aad is 0 bytes"): + ciphertext = "unhex('19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="''", message=plaintext) + + with When("aad is 1 byte"): + ciphertext = "unhex('19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'1'", message=plaintext) + + with When("aad is 256 bytes"): + ciphertext = "unhex('19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message=plaintext) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `decrypt` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there") + + with When("iv is specified using String with UTF8 characters"): + decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there") + + with When("iv is specified using FixedString type"): + decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there") + + with When("iv is specified using FixedString with UTF8 characters"): + decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `decrypt` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("key is specified using String with UTF8 characters"): + decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there") + + with When("key is specified using FixedString type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there") + + with When("key is specified using FixedString with UTF8 characters"): + decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `decrypt` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("mode is specified using FixedString type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `decrypt` functions returns String data type. + """ + ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(decrypt(" + mode + "," + ciphertext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get the return value"): + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `decrypt` function supports syntax + + ```sql + decrypt(ciphertext, key, mode, [iv, aad]) + ``` + """ + ciphertext = "19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614" + sql = f"SELECT decrypt('aes-128-gcm', unhex('{ciphertext}'), '0123456789123456', '012345678912', 'AAD')" + self.context.node.query(sql, step=When, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"), + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def decryption(self): + """Check that `decrypt` functions accepts `ciphertext` as the second parameter + and `mode` as the first parameter and we can convert the decrypted value into the original + value with the original data type. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + for datatype, plaintext in plaintexts: + + requirement = globals().get(f"""RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""", + requirements=[requirement]) as example: + + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + cast = None + endcast = None + ciphertext = f"unhex({ciphertext})" + compare = plaintext + + if datatype == "IPv4": + cast = "toIPv4(IPv4NumToString(reinterpretAsUInt32" + endcast = "))" + elif datatype in ["DateTime64", "UUID", "IPv6", "LowCardinality", "Enum8", "Enum16", "Decimal32", "Decimal64", "Decimal128", "Array"]: + xfail(reason="no conversion") + elif datatype == "NULL": + ciphertext = "NULL" + cast = "isNull" + compare = None + elif datatype in ["Float32", "Float64", "Date", "DateTime"] or "Int" in datatype: + cast = f"reinterpretAs{datatype}" + + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), + cast=cast, endcast=endcast, compare=compare, message="1") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Key("1.0") +) +def mismatched_key(self): + """Check that `decrypt` function returns garbage or an error when key parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + datatype = "String" + plaintext = "'1'" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched key"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_IV("1.0") +) +def mismatched_iv(self): + """Check that `decrypt` function returns garbage or an error when iv parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + datatype = "String" + plaintext = "'1'" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + if not iv_len: + continue + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched iv"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + iv=f"'a{iv[:iv_len-1]}'", + aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_AAD("1.0") +) +def mismatched_aad(self): + """Check that `decrypt` function returns garbage or an error when aad parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + datatype = "String" + plaintext = "'1'" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + if not aad_len: + continue + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched aad"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'a{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Mode("1.0") +) +def mismatched_mode(self): + """Check that `decrypt` function returns garbage or an error when mode parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8")) + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} datatype=utf8string iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + for mismatched_mode, _, _, _ in modes: + if mismatched_mode == mode: + continue + + with When(f"I decrypt using mismatched mode {mismatched_mode}"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 36 or 198"): + assert r.exitcode in [0, 36, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + condition = "Exception: Failed to decrypt" in output \ + or 'Exception: Invalid key size' in output \ + or output != plaintext + assert condition, error() + +@TestFeature +@Name("decrypt") +@Requirements( + RQ_SRS008_AES_Decrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `decrypt` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/decrypt_mysql.py b/tests/testflows/aes_encryption/tests/decrypt_mysql.py new file mode 100644 index 00000000000..b3df2121cd4 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/decrypt_mysql.py @@ -0,0 +1,521 @@ +# -*- coding: utf-8 -*- +import os +from importlib.machinery import SourceFileLoader + +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts.helpers import varname +from testflows.asserts import error + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def aes_decrypt_mysql(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, + step=When, cast=None, endcast=None, compare=None, no_checks=False): + """Execute `aes_decrypt_mysql` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if ciphertext is not None: + params.append(ciphertext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + if aad is not None: + params.append(aad) + + sql = f"aes_decrypt_mysql(" + ", ".join(params) + ")" + if cast: + sql = f"{cast}({sql}){endcast or ''}" + if compare: + sql = f"{compare} = {sql}" + sql = f"SELECT {sql}" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks) + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"), +) +def invalid_ciphertext(self): + """Check that `aes_decrypt_mysql` function does not crash when invalid + `ciphertext` is passed as the first parameter. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + invalid_ciphertexts = plaintexts + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len}"""): + d_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + for datatype, ciphertext in invalid_ciphertexts: + if datatype in ["NULL"]: + continue + with When(f"invalid ciphertext={ciphertext}"): + if "cfb" in mode or "ofb" in mode or "ctr" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, cast="hex") + else: + with When("I execute aes_decrypt_mysql function"): + r = aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, no_checks=True, step=By) + with Then("exitcode is not zero"): + assert r.exitcode in [198, 36] + with And("exception is present in the output"): + assert "DB::Exception:" in r.output + +@TestOutline(Scenario) +@Examples("mode", [ + ("'aes-128-gcm'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error("1.0"))), + ("'aes-192-gcm'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error("1.0"))), + ("'aes-256-gcm'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error("1.0"))), + ("'aes-128-ctr'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error("1.0"))), + ("'aes-192-ctr'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error("1.0"))), + ("'aes-256-ctr'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error("1.0"))), +]) +def unsupported_modes(self, mode): + """Check that `aes_decrypt_mysql` function returns an error when unsupported modes are specified. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + + aes_decrypt_mysql(ciphertext=ciphertext, mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `aes_decrypt_mysql` function returns an error when + we call it with invalid parameters. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + + with Example("no parameters"): + aes_decrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 0, expected 3 to 4") + + with Example("missing key and mode"): + aes_decrypt_mysql(ciphertext=ciphertext, exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 1") + + with Example("missing mode"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'123'", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 2") + + with Example("bad key type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=0, + message=None) + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0, + message=None) + + with Example("aad passed by mistake"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5") + + with Example("aad passed by mistake type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5") + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len", [ + # ECB + ("'aes-128-ecb'", 16, None, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s") +def key_or_iv_length_for_mode(self, mode, key_len, iv_len): + """Check that key or iv length for mode. + """ + ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3A')" + if mode == "'aes-128-ecb'": + ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3B')" + elif mode == "'aes-192-ecb'": + ciphertext = "unhex('073868ECDECA94133A61A0FFA282E877')" + elif mode == "'aes-256-ecb'": + ciphertext = "unhex('1729E5354D6EC44D89900ABDB09DC297')" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + if "ecb" in mode or "cbc" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + else: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, cast="hex") + + if iv_len is not None: + with When("iv is too short"): + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + if "ecb" in mode or "cbc" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + else: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, cast="hex") + else: + with When("iv is specified but not needed"): + if "ecb" in mode or "cbc" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + else: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode) + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `aes_decrypt_mysql` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there") + + with When("iv is specified using String with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there") + + with When("iv is specified using FixedString type"): + aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there") + + with When("iv is specified using FixedString with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `aes_decrypt` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("key is specified using String with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there") + + with When("key is specified using FixedString type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there") + + with When("key is specified using FixedString with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `aes_decrypt_mysql` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("mode is specified using FixedString type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `aes_decrypt_mysql` functions returns String data type. + """ + ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(aes_decrypt_mysql(" + mode + "," + ciphertext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get the return value"): + aes_decrypt_mysql(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `aes_decrypt_mysql` function supports syntax + + ```sql + aes_decrypt_mysql(ciphertext, key, mode, [iv]) + ``` + """ + ciphertext = "70FE78410D6EE237C2DE4A" + sql = f"SELECT aes_decrypt_mysql('aes-128-ofb', unhex('{ciphertext}'), '0123456789123456', '0123456789123456')" + self.context.node.query(sql, step=When, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def decryption(self): + """Check that `aes_decrypt_mysql` functions accepts `mode` as the first parameter + and `ciphertext` as the second parameter and we can convert the decrypted value into the original + value with the original data type. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + for datatype, plaintext in plaintexts: + + requirement = globals().get(f"""RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""", + requirements=[requirement]) as example: + + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + cast = None + endcast = None + ciphertext = f"unhex({ciphertext})" + compare = plaintext + + if datatype == "IPv4": + cast = "toIPv4(IPv4NumToString(reinterpretAsUInt32" + endcast = "))" + elif datatype in ["DateTime64", "UUID", "IPv6", "LowCardinality", "Enum8", "Enum16", "Decimal32", "Decimal64", "Decimal128", "Array"]: + xfail(reason="no conversion") + elif datatype == "NULL": + ciphertext = "NULL" + cast = "isNull" + compare = None + elif datatype in ["Float32", "Float64", "Date", "DateTime"] or "Int" in datatype: + cast = f"reinterpretAs{datatype}" + + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + cast=cast, endcast=endcast, compare=compare, message="1") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Key("1.0") +) +def mismatched_key(self): + """Check that `aes_decrypt_mysql` function returns garbage or an error when key parameter does not match. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched key"): + r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + cast="hex", no_checks=True) + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_IV("1.0") +) +def mismatched_iv(self): + """Check that `aes_decrypt_mysql` function returns garbage or an error when iv parameter does not match. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + if not iv_len: + continue + with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched key"): + r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + iv=f"'a{iv[:iv_len-1]}'", + cast="hex", no_checks=True) + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Mode("1.0") +) +def mismatched_mode(self): + """Check that `aes_decrypt_mysql` function returns garbage or an error when mode parameter does not match. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8")) + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + if not iv_len: + continue + + with Example(f"""mode={mode.strip("'")} datatype=utf8string key={key_len} iv={iv_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + for mismatched_mode, _, _ in mysql_modes: + if mismatched_mode == mode: + continue + + with When(f"I decrypt using a mismatched mode {mismatched_mode}"): + r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode, + iv=f"'{iv[:iv_len]}'", + cast="hex", no_checks=True) + + with Then("exitcode shoud be 0 or 36 or 198"): + assert r.exitcode in [0, 36, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != plaintext, error() + +@TestFeature +@Name("decrypt_mysql") +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `aes_decrypt_mysql` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) \ No newline at end of file diff --git a/tests/testflows/aes_encryption/tests/encrypt.py b/tests/testflows/aes_encryption/tests/encrypt.py new file mode 100644 index 00000000000..2fdd75fa0c8 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/encrypt.py @@ -0,0 +1,408 @@ +# -*- coding: utf-8 -*- +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts import values, error, snapshot + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def encrypt(self, plaintext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When): + """Execute `encrypt` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if plaintext is not None: + params.append(plaintext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + if aad is not None: + params.append(aad) + + sql = "SELECT hex(encrypt(" + ", ".join(params) + "))" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `encrypt` function returns an error when + we call it with invalid parameters. + """ + with Example("no parameters"): + encrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 0, expected 3 to 5") + + with Example("missing key and mode"): + encrypt(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 1") + + with Example("missing mode"): + encrypt(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 2") + + with Example("bad key type - UInt8"): + encrypt(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("bad aad type - UInt8"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode") + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # ECB + ("'aes-128-ecb'", 16, None, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), + # CTR + ("'aes-128-ctr'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ctr'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ctr'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the mode. + """ + plaintext = "'hello there'" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When("iv is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + if aad is None: + with When("aad is specified but not needed"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode") + + else: + with When("iv is specified but not needed"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'"))) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # GCM + ("'aes-128-gcm'", 16, 8, "'hello there aad'", + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-gcm'", 24, 8, "''", + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-gcm'", 32, 8, "'a'", + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the GCM mode. + """ + plaintext = "'hello there'" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When(f"iv is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=198, message="DB::Exception:") + else: + with When("iv is not specified"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + if aad is not None: + with When(f"aad is {aad}"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len]}'", aad=f"{aad}", mode=mode) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"), + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length("1.0") +) +def aad_parameter_types_and_length(self): + """Check that `encrypt` function accepts `aad` parameter as the fifth argument + of either `String` or `FixedString` types and that the length is not limited. + """ + plaintext = "'hello there'" + iv = "'012345678912'" + mode = "'aes-128-gcm'" + key = "'0123456789123456'" + + with When("aad is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'aad'", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526") + + with When("aad is specified using String with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39") + + with When("aad is specified using FixedString type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526") + + with When("aad is specified using FixedString with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39") + + with When("aad is 0 bytes"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="''", message="19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9") + + with When("aad is 1 byte"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'1'", message="19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035") + + with When("aad is 256 bytes"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message="19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `encrypt` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using String with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC") + + with When("iv is specified using FixedString type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using FixedString with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `encrypt` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using String with UTF8 characters"): + encrypt(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + + with When("key is specified using FixedString type"): + encrypt(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using FixedString with UTF8 characters"): + encrypt(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `encrypt` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("mode is specified using FixedString type"): + encrypt(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText("1.0"), + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def encryption(self): + """Check that `encrypt` functions accepts `plaintext` as the second parameter + with any data type and `mode` as the first parameter. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + for datatype, plaintext in plaintexts: + + requirement = globals().get(f"""RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""", + requirements=[requirement]) as example: + r = encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), aad=(None if not aad_len else f"'{aad}'")) + + with Then("I check output against snapshot"): + with values() as that: + example_name = basename(example.name) + assert that(snapshot(r.output.strip(), "encrypt", name=f"example_{example_name.replace(' ', '_')}")), error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `encrypt` functions returns String data type. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(encrypt(" + mode + "," + plaintext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get return ciphertext as hex"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `encrypt` function supports syntax + + ```sql + encrypt(plaintext, key, mode, [iv, aad]) + ``` + """ + sql = "SELECT hex(encrypt('aes-128-gcm', 'hello there', '0123456789123456', '012345678912', 'AAD'))" + self.context.node.query(sql, step=When, message="19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614") + +@TestFeature +@Name("encrypt") +@Requirements( + RQ_SRS008_AES_Encrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `encrypt` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/encrypt_mysql.py b/tests/testflows/aes_encryption/tests/encrypt_mysql.py new file mode 100644 index 00000000000..56183aff125 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/encrypt_mysql.py @@ -0,0 +1,326 @@ +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts import values, error, snapshot + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def aes_encrypt_mysql(self, plaintext=None, key=None, mode=None, iv=None, exitcode=0, message=None, step=When): + """Execute `aes_encrypt_mysql` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if plaintext is not None: + params.append(plaintext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + + sql = "SELECT hex(aes_encrypt_mysql(" + ", ".join(params) + "))" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Examples("mode", [ + ("'aes-128-gcm'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error("1.0"))), + ("'aes-192-gcm'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error("1.0"))), + ("'aes-256-gcm'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error("1.0"))), + ("'aes-128-ctr'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error("1.0"))), + ("'aes-192-ctr'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error("1.0"))), + ("'aes-256-ctr'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error("1.0"))), +]) +def unsupported_modes(self, mode): + """Check that `aes_encrypt_mysql` function returns an error when unsupported modes are specified. + """ + aes_encrypt_mysql(plaintext="'hello there'", mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `aes_encrypt_mysql` function returns an error when + we call it with invalid parameters. + """ + with Example("no parameters"): + aes_encrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt provided 0, expected 3 to 4") + + with Example("missing key and mode"): + aes_encrypt_mysql(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 1") + + with Example("missing mode"): + aes_encrypt_mysql(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 2") + + with Example("bad key type - UInt8"): + aes_encrypt_mysql(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0, + message=None) + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len", [ + # ECB + ("'aes-128-ecb'", 16, None, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s") +def key_or_iv_length_for_mode(self, mode, key_len, iv_len): + """Check that key or iv length for mode. + """ + plaintext = "'hello there'" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode) + + if iv_len is not None: + with When("iv is too short"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode) + else: + with When("iv is specified but not needed"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `aes_encrypt_mysql` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using String with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC") + + with When("iv is specified using FixedString type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using FixedString with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `aes_encrypt_mysql` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using String with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + + with When("key is specified using FixedString type"): + aes_encrypt_mysql(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using FixedString with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `aes_encrypt_mysql` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("mode is specified using FixedString type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `aes_encrypt_mysql` functions returns String data type. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(aes_encrypt_mysql("+ mode + "," + plaintext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get return ciphertext as hex"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `aes_encrypt_mysql` function supports syntax + + ```sql + aes_encrypt_mysql(plaintext, key, mode, [iv]) + ``` + """ + sql = "SELECT hex(aes_encrypt_mysql('aes-128-ofb', 'hello there', '0123456789123456', '0123456789123456'))" + self.context.node.query(sql, step=When, message="70FE78410D6EE237C2DE4A") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def encryption(self): + """Check that `aes_encrypt_mysql` functions accepts `plaintext` as the second parameter + with any data type and `mode` as the first parameter. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for mode, key_len, iv_len in mysql_modes: + for datatype, plaintext in plaintexts: + requirement = globals().get(f"""RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""", + requirements=[requirement]) as example: + + r = aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'")) + + with Then("I check output against snapshot"): + with values() as that: + example_name = basename(example.name) + assert that(snapshot(r.output.strip(), "encrypt_mysql", name=f"example_{example_name.replace(' ', '_')}")), error() + +@TestFeature +@Name("encrypt_mysql") +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `aes_encrypt_mysql` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot b/tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot new file mode 100644 index 00000000000..566c1074efb --- /dev/null +++ b/tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot @@ -0,0 +1,2700 @@ +example_mode_aes_128_ecb_datatype_String_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_FixedString_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_UInt8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_UInt16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_UInt32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_UInt64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Int8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Int16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_Int32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_Int64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Float32_iv_None_aad_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_ecb_datatype_Float64_iv_None_aad_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_ecb_datatype_Decimal32_iv_None_aad_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_ecb_datatype_Decimal64_iv_None_aad_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_ecb_datatype_Decimal128_iv_None_aad_None = r"""'5F3DBFA74809E45E03980357B26787AFF30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_UUID_iv_None_aad_None = r"""'FF9161B222B4A67481271035745F06D9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Date_iv_None_aad_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_ecb_datatype_DateTime_iv_None_aad_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_ecb_datatype_DateTime64_iv_None_aad_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_ecb_datatype_LowCardinality_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_Array_iv_None_aad_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_ecb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_ecb_datatype_IPv4_iv_None_aad_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_ecb_datatype_IPv6_iv_None_aad_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Enum8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Enum16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_192_ecb_datatype_String_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_FixedString_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_UInt8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_UInt16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_UInt32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_UInt64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Int8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Int16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_Int32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_Int64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Float32_iv_None_aad_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_ecb_datatype_Float64_iv_None_aad_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_ecb_datatype_Decimal32_iv_None_aad_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_ecb_datatype_Decimal64_iv_None_aad_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_ecb_datatype_Decimal128_iv_None_aad_None = r"""'79207931793E374FB5A3A2AC1ECA857AD8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_UUID_iv_None_aad_None = r"""'9FDB738E78D0D2F774C484ED82A854E4D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Date_iv_None_aad_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_ecb_datatype_DateTime_iv_None_aad_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_ecb_datatype_DateTime64_iv_None_aad_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_ecb_datatype_LowCardinality_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_Array_iv_None_aad_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_ecb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_ecb_datatype_IPv4_iv_None_aad_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_ecb_datatype_IPv6_iv_None_aad_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F232D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Enum8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Enum16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_256_ecb_datatype_String_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_FixedString_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_UInt8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_UInt16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_UInt32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_UInt64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Int8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Int16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_Int32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_Int64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Float32_iv_None_aad_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_ecb_datatype_Float64_iv_None_aad_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_ecb_datatype_Decimal32_iv_None_aad_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_ecb_datatype_Decimal64_iv_None_aad_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_ecb_datatype_Decimal128_iv_None_aad_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_UUID_iv_None_aad_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C34217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Date_iv_None_aad_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_ecb_datatype_DateTime_iv_None_aad_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_ecb_datatype_DateTime64_iv_None_aad_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_ecb_datatype_LowCardinality_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_Array_iv_None_aad_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_ecb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_ecb_datatype_IPv4_iv_None_aad_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_ecb_datatype_IPv6_iv_None_aad_None = r"""'8CBF2DC164F4086B8DD14B75E3065621217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Enum8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Enum16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_128_cbc_datatype_String_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_FixedString_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_UInt8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_UInt16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_UInt32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_UInt64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Int8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Int16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_Int32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_Int64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Float32_iv_None_aad_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_cbc_datatype_Float64_iv_None_aad_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_cbc_datatype_Decimal32_iv_None_aad_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_cbc_datatype_Decimal64_iv_None_aad_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_cbc_datatype_Decimal128_iv_None_aad_None = r"""'5F3DBFA74809E45E03980357B26787AF0D55B905F5525D3F5916FF811D8A6E7E'""" + +example_mode_aes_128_cbc_datatype_UUID_iv_None_aad_None = r"""'FF9161B222B4A67481271035745F06D991B6833DF67CBA9BC6E1AAEADBE363BB'""" + +example_mode_aes_128_cbc_datatype_Date_iv_None_aad_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_cbc_datatype_DateTime_iv_None_aad_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_cbc_datatype_DateTime64_iv_None_aad_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_Array_iv_None_aad_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_cbc_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_iv_None_aad_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_cbc_datatype_IPv6_iv_None_aad_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9B1B4BE8B15BAE0B2C9196D69E3D53C6C'""" + +example_mode_aes_128_cbc_datatype_Enum8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Enum16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_192_cbc_datatype_String_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_FixedString_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_UInt8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_UInt16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_UInt32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_UInt64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Int8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Int16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_Int32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_Int64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Float32_iv_None_aad_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_cbc_datatype_Float64_iv_None_aad_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_cbc_datatype_Decimal32_iv_None_aad_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_cbc_datatype_Decimal64_iv_None_aad_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_cbc_datatype_Decimal128_iv_None_aad_None = r"""'79207931793E374FB5A3A2AC1ECA857A583603B3047000A843425EECA4C35311'""" + +example_mode_aes_192_cbc_datatype_UUID_iv_None_aad_None = r"""'9FDB738E78D0D2F774C484ED82A854E46B580C61DBE08478DC523DA6AD605078'""" + +example_mode_aes_192_cbc_datatype_Date_iv_None_aad_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_cbc_datatype_DateTime_iv_None_aad_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_cbc_datatype_DateTime64_iv_None_aad_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_Array_iv_None_aad_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_cbc_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_iv_None_aad_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_cbc_datatype_IPv6_iv_None_aad_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F2326879326F3973401A6293A92BCB8EDFC4'""" + +example_mode_aes_192_cbc_datatype_Enum8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Enum16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_256_cbc_datatype_String_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_FixedString_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_UInt8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_UInt16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_UInt32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_UInt64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Int8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Int16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_Int32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_Int64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Float32_iv_None_aad_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_cbc_datatype_Float64_iv_None_aad_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_cbc_datatype_Decimal32_iv_None_aad_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_cbc_datatype_Decimal64_iv_None_aad_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_cbc_datatype_Decimal128_iv_None_aad_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB36585C228B0286E7A8D8DBAF754C4C38'""" + +example_mode_aes_256_cbc_datatype_UUID_iv_None_aad_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C3428548E6132D69A22B320775A21ABA11F'""" + +example_mode_aes_256_cbc_datatype_Date_iv_None_aad_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_cbc_datatype_DateTime_iv_None_aad_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_cbc_datatype_DateTime64_iv_None_aad_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_Array_iv_None_aad_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_cbc_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_iv_None_aad_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_cbc_datatype_IPv6_iv_None_aad_None = r"""'8CBF2DC164F4086B8DD14B75E3065621393DE8421BAA5AE5E87096AEA7087507'""" + +example_mode_aes_256_cbc_datatype_Enum8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Enum16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_128_cbc_datatype_String_iv_16_aad_None = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_FixedString_iv_16_aad_None = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_UInt8_iv_16_aad_None = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_UInt16_iv_16_aad_None = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_UInt32_iv_16_aad_None = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_UInt64_iv_16_aad_None = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Int8_iv_16_aad_None = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Int16_iv_16_aad_None = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_Int32_iv_16_aad_None = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_Int64_iv_16_aad_None = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Float32_iv_16_aad_None = r"""'A5425BDEB6B83E311C45249DAF3153F5'""" + +example_mode_aes_128_cbc_datatype_Float64_iv_16_aad_None = r"""'EEDA98EC4045C7D351F3905313073B79'""" + +example_mode_aes_128_cbc_datatype_Decimal32_iv_16_aad_None = r"""'52EBB74292ECD37A29E9809166CC77DB'""" + +example_mode_aes_128_cbc_datatype_Decimal64_iv_16_aad_None = r"""'95EF455767EC8FBD32BAAEFFB44FEEB7'""" + +example_mode_aes_128_cbc_datatype_Decimal128_iv_16_aad_None = r"""'94C066884FA09B0D3C750F20A2823304A2FE20B6B69AB18373E3F58623E0D7FB'""" + +example_mode_aes_128_cbc_datatype_UUID_iv_16_aad_None = r"""'1D909C15BB882E89AD68B1EFEAC72148DCD05E2303B6BE19007A945AFB778B42'""" + +example_mode_aes_128_cbc_datatype_Date_iv_16_aad_None = r"""'24A4F8CE8A9FAE48A0AFEB8A6203EFEA'""" + +example_mode_aes_128_cbc_datatype_DateTime_iv_16_aad_None = r"""'0DD5554819E3995B1B6B00362AEE9424'""" + +example_mode_aes_128_cbc_datatype_DateTime64_iv_16_aad_None = r"""'0E55319903957C9D1FDA4FB65C3871CB'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_iv_16_aad_None = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_Array_iv_16_aad_None = r"""'D53C82A5D13256B88DF41C1C1D924E40'""" + +example_mode_aes_128_cbc_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_iv_16_aad_None = r"""'C0D81AAB3134EAB5B1F190958C6A29F9'""" + +example_mode_aes_128_cbc_datatype_IPv6_iv_16_aad_None = r"""'AE1A36F75C9BB387121445069A9968CA247FA4459ED3C8809089FEE334EB1EC7'""" + +example_mode_aes_128_cbc_datatype_Enum8_iv_16_aad_None = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Enum16_iv_16_aad_None = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_192_cbc_datatype_String_iv_16_aad_None = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_FixedString_iv_16_aad_None = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_UInt8_iv_16_aad_None = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_UInt16_iv_16_aad_None = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_UInt32_iv_16_aad_None = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_UInt64_iv_16_aad_None = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Int8_iv_16_aad_None = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Int16_iv_16_aad_None = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_Int32_iv_16_aad_None = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_Int64_iv_16_aad_None = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Float32_iv_16_aad_None = r"""'62EBE4FD1035D405BBD6C41436780E13'""" + +example_mode_aes_192_cbc_datatype_Float64_iv_16_aad_None = r"""'5706FC9892A4C1AB48FC93E13C9C72FE'""" + +example_mode_aes_192_cbc_datatype_Decimal32_iv_16_aad_None = r"""'BB056843D369A5E55982C92AD52EEC07'""" + +example_mode_aes_192_cbc_datatype_Decimal64_iv_16_aad_None = r"""'70ACD4156F9AC1444A75EFCB9202CA00'""" + +example_mode_aes_192_cbc_datatype_Decimal128_iv_16_aad_None = r"""'04748A45840A0CAAC83F139DB01C504B01FC56631A8B2FFBE68F2FC85B6FEEDE'""" + +example_mode_aes_192_cbc_datatype_UUID_iv_16_aad_None = r"""'D7B2ABC08F67823F61C3E8F680C12B3A8AA3E3711D412CB55ACFBC89C14949A8'""" + +example_mode_aes_192_cbc_datatype_Date_iv_16_aad_None = r"""'734BBE526E56B280E90E53DDEA7DB69B'""" + +example_mode_aes_192_cbc_datatype_DateTime_iv_16_aad_None = r"""'9B9BE7CC20F75DA3F39F688DE3A1ADAA'""" + +example_mode_aes_192_cbc_datatype_DateTime64_iv_16_aad_None = r"""'554FCAAF985378A561F7C6ED91E20C89'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_iv_16_aad_None = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_Array_iv_16_aad_None = r"""'D85AF1078F110329896EFC462340171E'""" + +example_mode_aes_192_cbc_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_iv_16_aad_None = r"""'6AF45078B1E924B6C107D4C0236EA937'""" + +example_mode_aes_192_cbc_datatype_IPv6_iv_16_aad_None = r"""'9E4F8E54B265A340090DC7FE4F53BB50048442F5632A7B1630AE80DFD938E9AA'""" + +example_mode_aes_192_cbc_datatype_Enum8_iv_16_aad_None = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Enum16_iv_16_aad_None = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_256_cbc_datatype_String_iv_16_aad_None = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_FixedString_iv_16_aad_None = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_UInt8_iv_16_aad_None = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_UInt16_iv_16_aad_None = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_UInt32_iv_16_aad_None = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_UInt64_iv_16_aad_None = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Int8_iv_16_aad_None = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Int16_iv_16_aad_None = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_Int32_iv_16_aad_None = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_Int64_iv_16_aad_None = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Float32_iv_16_aad_None = r"""'E639AA3E45D8C2759181FD736CD58EDC'""" + +example_mode_aes_256_cbc_datatype_Float64_iv_16_aad_None = r"""'CFEF3FDC054997559DF5DCFB5F215B58'""" + +example_mode_aes_256_cbc_datatype_Decimal32_iv_16_aad_None = r"""'E2F57A092A1759D39F4AE67C9543FAB8'""" + +example_mode_aes_256_cbc_datatype_Decimal64_iv_16_aad_None = r"""'6259A2CFD3D83352A44C03DB050077B3'""" + +example_mode_aes_256_cbc_datatype_Decimal128_iv_16_aad_None = r"""'AEC71CA2D87098392689F9EB2ED93A84FA5787E643E28CB3C2013F8FCC24E387'""" + +example_mode_aes_256_cbc_datatype_UUID_iv_16_aad_None = r"""'88BA86B14A468DC92084B7152B172E142D88CBFB639A8FF2F480F1727972251C'""" + +example_mode_aes_256_cbc_datatype_Date_iv_16_aad_None = r"""'C67C84B1C6BF4527A7E730499FF39C86'""" + +example_mode_aes_256_cbc_datatype_DateTime_iv_16_aad_None = r"""'7FDC1B0797A5F3C04CDA82729A1EA4AA'""" + +example_mode_aes_256_cbc_datatype_DateTime64_iv_16_aad_None = r"""'B1B7401FB2B65BCB3448C1BE179F6AA6'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_iv_16_aad_None = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_Array_iv_16_aad_None = r"""'6BB1E8429CC612B0AA74282B81D4FE8A'""" + +example_mode_aes_256_cbc_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_iv_16_aad_None = r"""'51364C8DC6882CA1F03CF7FB45117EEF'""" + +example_mode_aes_256_cbc_datatype_IPv6_iv_16_aad_None = r"""'87A1C4D4672EFE64DC98E040EAD6B3126C899C263577B3D8EE8A3952BE5CDC1B'""" + +example_mode_aes_256_cbc_datatype_Enum8_iv_16_aad_None = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Enum16_iv_16_aad_None = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_128_cfb1_datatype_String_iv_None_aad_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_FixedString_iv_None_aad_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_UInt8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_iv_None_aad_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_UInt32_iv_None_aad_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_UInt64_iv_None_aad_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Int8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_iv_None_aad_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_Int32_iv_None_aad_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_Int64_iv_None_aad_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Float32_iv_None_aad_None = r"""'0000B9AB'""" + +example_mode_aes_128_cfb1_datatype_Float64_iv_None_aad_None = r"""'000000000000FFF6'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_iv_None_aad_None = r"""'2E09CA6A'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_iv_None_aad_None = r"""'2E09CA6A6DBEE799'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_iv_None_aad_None = r"""'2E09CA6A6DBEE79923BA65C6B78FD199'""" + +example_mode_aes_128_cfb1_datatype_UUID_iv_None_aad_None = r"""'E590DFB515D3A518F85C66A6A5EC9C6E'""" + +example_mode_aes_128_cfb1_datatype_Date_iv_None_aad_None = r"""'42F0'""" + +example_mode_aes_128_cfb1_datatype_DateTime_iv_None_aad_None = r"""'5475EC3D'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_iv_None_aad_None = r"""'21CDF1128AE44A37'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_iv_None_aad_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_Array_iv_None_aad_None = r"""'0170'""" + +example_mode_aes_128_cfb1_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_iv_None_aad_None = r"""'240A9E43'""" + +example_mode_aes_128_cfb1_datatype_IPv6_iv_None_aad_None = r"""'2E642EF4B07D9B1251BE3B3CBDBCC6F6'""" + +example_mode_aes_128_cfb1_datatype_Enum8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_iv_None_aad_None = r"""'0173'""" + +example_mode_aes_192_cfb1_datatype_String_iv_None_aad_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_FixedString_iv_None_aad_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_UInt8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_UInt16_iv_None_aad_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_UInt32_iv_None_aad_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_UInt64_iv_None_aad_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Int8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Int16_iv_None_aad_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_Int32_iv_None_aad_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_Int64_iv_None_aad_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Float32_iv_None_aad_None = r"""'0000FCAE'""" + +example_mode_aes_192_cfb1_datatype_Float64_iv_None_aad_None = r"""'000000000000A79C'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_iv_None_aad_None = r"""'3F406C3F'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_iv_None_aad_None = r"""'3F406C3F3A41B134'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_iv_None_aad_None = r"""'3F406C3F3A41B134310D6B68BEBC5708'""" + +example_mode_aes_192_cfb1_datatype_UUID_iv_None_aad_None = r"""'B7F80F1BDCA1C4193E5AB11078FEA213'""" + +example_mode_aes_192_cfb1_datatype_Date_iv_None_aad_None = r"""'6FF6'""" + +example_mode_aes_192_cfb1_datatype_DateTime_iv_None_aad_None = r"""'7013E555'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_iv_None_aad_None = r"""'371AF0291536F5B7'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_iv_None_aad_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_Array_iv_None_aad_None = r"""'01FA'""" + +example_mode_aes_192_cfb1_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_iv_None_aad_None = r"""'33895F70'""" + +example_mode_aes_192_cfb1_datatype_IPv6_iv_None_aad_None = r"""'3F24552946522B931290F904186B055A'""" + +example_mode_aes_192_cfb1_datatype_Enum8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Enum16_iv_None_aad_None = r"""'01F9'""" + +example_mode_aes_256_cfb1_datatype_String_iv_None_aad_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_FixedString_iv_None_aad_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_UInt8_iv_None_aad_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_UInt16_iv_None_aad_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_UInt32_iv_None_aad_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_UInt64_iv_None_aad_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Int8_iv_None_aad_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Int16_iv_None_aad_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_Int32_iv_None_aad_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_Int64_iv_None_aad_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Float32_iv_None_aad_None = r"""'B85F0E63'""" + +example_mode_aes_256_cfb1_datatype_Float64_iv_None_aad_None = r"""'B85FDB5A8FE0C0BB'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_iv_None_aad_None = r"""'891B85B3'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_iv_None_aad_None = r"""'891B85B3C1BA6EE1'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_iv_None_aad_None = r"""'891B85B3C1BA6EE137EF658F618D1F3F'""" + +example_mode_aes_256_cfb1_datatype_UUID_iv_None_aad_None = r"""'121B5EE9929417BC1CDBDB390BC93B4A'""" + +example_mode_aes_256_cfb1_datatype_Date_iv_None_aad_None = r"""'D40F'""" + +example_mode_aes_256_cfb1_datatype_DateTime_iv_None_aad_None = r"""'CF27297C'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_iv_None_aad_None = r"""'8773F350CD394D36'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_iv_None_aad_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_Array_iv_None_aad_None = r"""'B9EE'""" + +example_mode_aes_256_cfb1_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_iv_None_aad_None = r"""'8383FD3C'""" + +example_mode_aes_256_cfb1_datatype_IPv6_iv_None_aad_None = r"""'897A84A02FD451D3DDB92FF290BF9B7C'""" + +example_mode_aes_256_cfb1_datatype_Enum8_iv_None_aad_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Enum16_iv_None_aad_None = r"""'B9ED'""" + +example_mode_aes_128_cfb1_datatype_String_iv_16_aad_None = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_FixedString_iv_16_aad_None = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_UInt8_iv_16_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_iv_16_aad_None = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_UInt32_iv_16_aad_None = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_UInt64_iv_16_aad_None = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Int8_iv_16_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_iv_16_aad_None = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_Int32_iv_16_aad_None = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_Int64_iv_16_aad_None = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Float32_iv_16_aad_None = r"""'00B931F2'""" + +example_mode_aes_128_cfb1_datatype_Float64_iv_16_aad_None = r"""'00B99AAE199C3C93'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_iv_16_aad_None = r"""'2D557511'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_iv_16_aad_None = r"""'2D557511511F90FB'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_iv_16_aad_None = r"""'2D557511511F90FBC464352E8A02FC51'""" + +example_mode_aes_128_cfb1_datatype_UUID_iv_16_aad_None = r"""'8AE269086C72AD682EB92ABA6CA58E49'""" + +example_mode_aes_128_cfb1_datatype_Date_iv_16_aad_None = r"""'5FC9'""" + +example_mode_aes_128_cfb1_datatype_DateTime_iv_16_aad_None = r"""'42970865'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_iv_16_aad_None = r"""'20B310A2F7EF8460'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_iv_16_aad_None = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_Array_iv_16_aad_None = r"""'018A'""" + +example_mode_aes_128_cfb1_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_iv_16_aad_None = r"""'27476DAF'""" + +example_mode_aes_128_cfb1_datatype_IPv6_iv_16_aad_None = r"""'2D311FBDC0A5C652AAD863398F94C5C3'""" + +example_mode_aes_128_cfb1_datatype_Enum8_iv_16_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_iv_16_aad_None = r"""'0188'""" + +example_mode_aes_192_cfb1_datatype_String_iv_16_aad_None = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_FixedString_iv_16_aad_None = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_UInt8_iv_16_aad_None = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_UInt16_iv_16_aad_None = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_UInt32_iv_16_aad_None = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_UInt64_iv_16_aad_None = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Int8_iv_16_aad_None = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Int16_iv_16_aad_None = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_Int32_iv_16_aad_None = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_Int64_iv_16_aad_None = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Float32_iv_16_aad_None = r"""'07955BCF'""" + +example_mode_aes_192_cfb1_datatype_Float64_iv_16_aad_None = r"""'0795A57CA222A36E'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_iv_16_aad_None = r"""'2A15BB86'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_iv_16_aad_None = r"""'2A15BB86FB961E7D'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_iv_16_aad_None = r"""'2A15BB86FB961E7D0DD5055987176AF4'""" + +example_mode_aes_192_cfb1_datatype_UUID_iv_16_aad_None = r"""'DA2338793C7B9E0F6722E272062F5EA1'""" + +example_mode_aes_192_cfb1_datatype_Date_iv_16_aad_None = r"""'4AAB'""" + +example_mode_aes_192_cfb1_datatype_DateTime_iv_16_aad_None = r"""'5B6A8EE6'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_iv_16_aad_None = r"""'23C4E2A707F73EF4'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_iv_16_aad_None = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_Array_iv_16_aad_None = r"""'069C'""" + +example_mode_aes_192_cfb1_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_iv_16_aad_None = r"""'2470A839'""" + +example_mode_aes_192_cfb1_datatype_IPv6_iv_16_aad_None = r"""'2A712A746781131B2DC4EB92E31C72FA'""" + +example_mode_aes_192_cfb1_datatype_Enum8_iv_16_aad_None = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Enum16_iv_16_aad_None = r"""'069E'""" + +example_mode_aes_256_cfb1_datatype_String_iv_16_aad_None = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_FixedString_iv_16_aad_None = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_UInt8_iv_16_aad_None = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_UInt16_iv_16_aad_None = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_UInt32_iv_16_aad_None = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_UInt64_iv_16_aad_None = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Int8_iv_16_aad_None = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Int16_iv_16_aad_None = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_Int32_iv_16_aad_None = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_Int64_iv_16_aad_None = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Float32_iv_16_aad_None = r"""'7F630BBA'""" + +example_mode_aes_256_cfb1_datatype_Float64_iv_16_aad_None = r"""'7F638DFAAA434E6B'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_iv_16_aad_None = r"""'4F430FBA'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_iv_16_aad_None = r"""'4F430FBAA3AAF884'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_iv_16_aad_None = r"""'4F430FBAA3AAF8845DB7BBA7F98F49C4'""" + +example_mode_aes_256_cfb1_datatype_UUID_iv_16_aad_None = r"""'B06F4A8C3BF3A8D32D113D0D40397C8F'""" + +example_mode_aes_256_cfb1_datatype_Date_iv_16_aad_None = r"""'30CE'""" + +example_mode_aes_256_cfb1_datatype_DateTime_iv_16_aad_None = r"""'206545FA'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_iv_16_aad_None = r"""'43756F28C68E3D55'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_iv_16_aad_None = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_Array_iv_16_aad_None = r"""'7EA3'""" + +example_mode_aes_256_cfb1_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_iv_16_aad_None = r"""'4526FCCF'""" + +example_mode_aes_256_cfb1_datatype_IPv6_iv_16_aad_None = r"""'4F23BDAC741DB8767CE6AE24888545A2'""" + +example_mode_aes_256_cfb1_datatype_Enum8_iv_16_aad_None = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Enum16_iv_16_aad_None = r"""'7EA1'""" + +example_mode_aes_128_cfb8_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_UInt16_iv_None_aad_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_UInt32_iv_None_aad_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_UInt64_iv_None_aad_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Int16_iv_None_aad_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_Int32_iv_None_aad_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_Int64_iv_None_aad_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Float32_iv_None_aad_None = r"""'10671940'""" + +example_mode_aes_128_cfb8_datatype_Float64_iv_None_aad_None = r"""'106799607DBF56DA'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_iv_None_aad_None = r"""'30756C94'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_iv_None_aad_None = r"""'30756C9417D3C023'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_iv_None_aad_None = r"""'30756C9417D3C023705550B7BEF872FF'""" + +example_mode_aes_128_cfb8_datatype_UUID_iv_None_aad_None = r"""'F7FE50CF0647659CB0D401B5C0E259D3'""" + +example_mode_aes_128_cfb8_datatype_Date_iv_None_aad_None = r"""'46EA'""" + +example_mode_aes_128_cfb8_datatype_DateTime_iv_None_aad_None = r"""'5EB4905E'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_iv_None_aad_None = r"""'3BB70F8E64D7C6A7'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_Array_iv_None_aad_None = r"""'11FD'""" + +example_mode_aes_128_cfb8_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_iv_None_aad_None = r"""'3DC2BE9E'""" + +example_mode_aes_128_cfb8_datatype_IPv6_iv_None_aad_None = r"""'303ABAC6F4F380D9F077DFC79C82D1A1'""" + +example_mode_aes_128_cfb8_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Enum16_iv_None_aad_None = r"""'11FF'""" + +example_mode_aes_192_cfb8_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_UInt16_iv_None_aad_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_UInt32_iv_None_aad_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_UInt64_iv_None_aad_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Int16_iv_None_aad_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_Int32_iv_None_aad_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_Int64_iv_None_aad_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Float32_iv_None_aad_None = r"""'07EDB300'""" + +example_mode_aes_192_cfb8_datatype_Float64_iv_None_aad_None = r"""'07ED3359B91DEF3B'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_iv_None_aad_None = r"""'275947FE'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_iv_None_aad_None = r"""'275947FE4B3390EE'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_iv_None_aad_None = r"""'275947FE4B3390EE7A2541BC8E2F58D7'""" + +example_mode_aes_192_cfb8_datatype_UUID_iv_None_aad_None = r"""'E0C082C032FB8ED756F9345E270A283B'""" + +example_mode_aes_192_cfb8_datatype_Date_iv_None_aad_None = r"""'5109'""" + +example_mode_aes_192_cfb8_datatype_DateTime_iv_None_aad_None = r"""'49713150'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_iv_None_aad_None = r"""'2C10FB4FEC471EF7'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_Array_iv_None_aad_None = r"""'0681'""" + +example_mode_aes_192_cfb8_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_iv_None_aad_None = r"""'2A41C8F2'""" + +example_mode_aes_192_cfb8_datatype_IPv6_iv_None_aad_None = r"""'271682C9379C5A46C68488DC33D0C278'""" + +example_mode_aes_192_cfb8_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Enum16_iv_None_aad_None = r"""'0683'""" + +example_mode_aes_256_cfb8_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_UInt16_iv_None_aad_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_UInt32_iv_None_aad_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_UInt64_iv_None_aad_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Int16_iv_None_aad_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_Int32_iv_None_aad_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_Int64_iv_None_aad_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Float32_iv_None_aad_None = r"""'B05A05BE'""" + +example_mode_aes_256_cfb8_datatype_Float64_iv_None_aad_None = r"""'B05A8510DB2F16A0'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_iv_None_aad_None = r"""'906B5777'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_iv_None_aad_None = r"""'906B57771CB81F37'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_iv_None_aad_None = r"""'906B57771CB81F378D932AE788527DE2'""" + +example_mode_aes_256_cfb8_datatype_UUID_iv_None_aad_None = r"""'57FB06BA6F4BA51D7A61D65A7827A18D'""" + +example_mode_aes_256_cfb8_datatype_Date_iv_None_aad_None = r"""'E652'""" + +example_mode_aes_256_cfb8_datatype_DateTime_iv_None_aad_None = r"""'FEEEADA4'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_iv_None_aad_None = r"""'9BB36DEF05FF5975'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_Array_iv_None_aad_None = r"""'B15D'""" + +example_mode_aes_256_cfb8_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_iv_None_aad_None = r"""'9DC836F3'""" + +example_mode_aes_256_cfb8_datatype_IPv6_iv_None_aad_None = r"""'90242F0083C8B0221DF3B5755EC8D99C'""" + +example_mode_aes_256_cfb8_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Enum16_iv_None_aad_None = r"""'B15F'""" + +example_mode_aes_128_cfb8_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_UInt16_iv_16_aad_None = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_UInt32_iv_16_aad_None = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_UInt64_iv_16_aad_None = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Int16_iv_16_aad_None = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_Int32_iv_16_aad_None = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_Int64_iv_16_aad_None = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Float32_iv_16_aad_None = r"""'3232B23D'""" + +example_mode_aes_128_cfb8_datatype_Float64_iv_16_aad_None = r"""'323232323232C2A6'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_iv_16_aad_None = r"""'12ABA873'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_iv_16_aad_None = r"""'12ABA873E2E24473'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_iv_16_aad_None = r"""'12ABA873E2E24473166434D82270A19C'""" + +example_mode_aes_128_cfb8_datatype_UUID_iv_16_aad_None = r"""'D529D970A38CCB794F856E4458D0E2D4'""" + +example_mode_aes_128_cfb8_datatype_Date_iv_16_aad_None = r"""'6445'""" + +example_mode_aes_128_cfb8_datatype_DateTime_iv_16_aad_None = r"""'7CBF2FDA'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_iv_16_aad_None = r"""'191C7B5A063F562D'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_Array_iv_16_aad_None = r"""'336A'""" + +example_mode_aes_128_cfb8_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_iv_16_aad_None = r"""'1F0A367A'""" + +example_mode_aes_128_cfb8_datatype_IPv6_iv_16_aad_None = r"""'12E4B19D97DC9F2C61A671C51B1201D2'""" + +example_mode_aes_128_cfb8_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Enum16_iv_16_aad_None = r"""'3368'""" + +example_mode_aes_192_cfb8_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_UInt16_iv_16_aad_None = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_UInt32_iv_16_aad_None = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_UInt64_iv_16_aad_None = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Int16_iv_16_aad_None = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_Int32_iv_16_aad_None = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_Int64_iv_16_aad_None = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Float32_iv_16_aad_None = r"""'6861DF7A'""" + +example_mode_aes_192_cfb8_datatype_Float64_iv_16_aad_None = r"""'68615FBC184B8D50'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_iv_16_aad_None = r"""'48041B5C'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_iv_16_aad_None = r"""'48041B5C6BEF70DD'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_iv_16_aad_None = r"""'48041B5C6BEF70DD4CDABC1FC8C2C684'""" + +example_mode_aes_192_cfb8_datatype_UUID_iv_16_aad_None = r"""'8FF1142976A9808C0F475A3D2A34D06D'""" + +example_mode_aes_192_cfb8_datatype_Date_iv_16_aad_None = r"""'3E6D'""" + +example_mode_aes_192_cfb8_datatype_DateTime_iv_16_aad_None = r"""'269AFDC7'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_iv_16_aad_None = r"""'4350703E05F43A50'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_Array_iv_16_aad_None = r"""'6926'""" + +example_mode_aes_192_cfb8_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_iv_16_aad_None = r"""'45979A4C'""" + +example_mode_aes_192_cfb8_datatype_IPv6_iv_16_aad_None = r"""'484BFA49756D837181B7EE03EBCF2B79'""" + +example_mode_aes_192_cfb8_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Enum16_iv_16_aad_None = r"""'6924'""" + +example_mode_aes_256_cfb8_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_UInt16_iv_16_aad_None = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_UInt32_iv_16_aad_None = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_UInt64_iv_16_aad_None = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Int16_iv_16_aad_None = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_Int32_iv_16_aad_None = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_Int64_iv_16_aad_None = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Float32_iv_16_aad_None = r"""'69B127F9'""" + +example_mode_aes_256_cfb8_datatype_Float64_iv_16_aad_None = r"""'69B1A72CB81A0FFF'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_iv_16_aad_None = r"""'49378750'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_iv_16_aad_None = r"""'493787505DFF5606'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_iv_16_aad_None = r"""'493787505DFF5606774649C631E6E0E7'""" + +example_mode_aes_256_cfb8_datatype_UUID_iv_16_aad_None = r"""'8E09A60AA21565C888B2D92942896930'""" + +example_mode_aes_256_cfb8_datatype_Date_iv_16_aad_None = r"""'3FF1'""" + +example_mode_aes_256_cfb8_datatype_DateTime_iv_16_aad_None = r"""'274E13D8'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_iv_16_aad_None = r"""'4211DFF611769F37'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_Array_iv_16_aad_None = r"""'682E'""" + +example_mode_aes_256_cfb8_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_iv_16_aad_None = r"""'442DB771'""" + +example_mode_aes_256_cfb8_datatype_IPv6_iv_16_aad_None = r"""'4978AF3EED91F4AD14F7C326CCD96804'""" + +example_mode_aes_256_cfb8_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Enum16_iv_16_aad_None = r"""'682C'""" + +example_mode_aes_128_cfb128_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_UInt16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_UInt32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_UInt64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Int16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_Int32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_Int64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Float32_iv_None_aad_None = r"""'10DF418A'""" + +example_mode_aes_128_cfb128_datatype_Float64_iv_None_aad_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_iv_None_aad_None = r"""'3091C1B5'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_iv_None_aad_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_iv_None_aad_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_cfb128_datatype_UUID_iv_None_aad_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_cfb128_datatype_Date_iv_None_aad_None = r"""'4698'""" + +example_mode_aes_128_cfb128_datatype_DateTime_iv_None_aad_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_iv_None_aad_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_Array_iv_None_aad_None = r"""'11DD'""" + +example_mode_aes_128_cfb128_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_iv_None_aad_None = r"""'3D5D201E'""" + +example_mode_aes_128_cfb128_datatype_IPv6_iv_None_aad_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_cfb128_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Enum16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_192_cfb128_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_UInt16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_UInt64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Int16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_Int32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_Int64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Float32_iv_None_aad_None = r"""'07B799A2'""" + +example_mode_aes_192_cfb128_datatype_Float64_iv_None_aad_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_iv_None_aad_None = r"""'27F9199D'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_iv_None_aad_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_iv_None_aad_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_cfb128_datatype_UUID_iv_None_aad_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_cfb128_datatype_Date_iv_None_aad_None = r"""'51F0'""" + +example_mode_aes_192_cfb128_datatype_DateTime_iv_None_aad_None = r"""'496715C3'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_iv_None_aad_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_Array_iv_None_aad_None = r"""'06B5'""" + +example_mode_aes_192_cfb128_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_iv_None_aad_None = r"""'2A35F836'""" + +example_mode_aes_192_cfb128_datatype_IPv6_iv_None_aad_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_cfb128_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Enum16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_256_cfb128_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_UInt16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_UInt32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_UInt64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Int16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_Int32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_Int64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Float32_iv_None_aad_None = r"""'B08E4FA1'""" + +example_mode_aes_256_cfb128_datatype_Float64_iv_None_aad_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_iv_None_aad_None = r"""'90C0CF9E'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_cfb128_datatype_UUID_iv_None_aad_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_cfb128_datatype_Date_iv_None_aad_None = r"""'E6C9'""" + +example_mode_aes_256_cfb128_datatype_DateTime_iv_None_aad_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_iv_None_aad_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_Array_iv_None_aad_None = r"""'B18C'""" + +example_mode_aes_256_cfb128_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_iv_None_aad_None = r"""'9D0C2E35'""" + +example_mode_aes_256_cfb128_datatype_IPv6_iv_None_aad_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_cfb128_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Enum16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_128_cfb128_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_UInt16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_UInt32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_UInt64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Int16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_Int32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_Int64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Float32_iv_16_aad_None = r"""'328829BB'""" + +example_mode_aes_128_cfb128_datatype_Float64_iv_16_aad_None = r"""'3288A984DD060F67'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_iv_16_aad_None = r"""'12C6A984'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_iv_16_aad_None = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_iv_16_aad_None = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_cfb128_datatype_UUID_iv_16_aad_None = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_cfb128_datatype_Date_iv_16_aad_None = r"""'64CF'""" + +example_mode_aes_128_cfb128_datatype_DateTime_iv_16_aad_None = r"""'7C58A5DA'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_iv_16_aad_None = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_Array_iv_16_aad_None = r"""'338A'""" + +example_mode_aes_128_cfb128_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_iv_16_aad_None = r"""'1F0A482F'""" + +example_mode_aes_128_cfb128_datatype_IPv6_iv_16_aad_None = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_cfb128_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Enum16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_192_cfb128_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_UInt16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_UInt64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Int16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_Int32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_Int64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Float32_iv_16_aad_None = r"""'68C767AD'""" + +example_mode_aes_192_cfb128_datatype_Float64_iv_16_aad_None = r"""'68C7E792B710878E'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_iv_16_aad_None = r"""'4889E792'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_iv_16_aad_None = r"""'4889E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_iv_16_aad_None = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_cfb128_datatype_UUID_iv_16_aad_None = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_cfb128_datatype_Date_iv_16_aad_None = r"""'3E80'""" + +example_mode_aes_192_cfb128_datatype_DateTime_iv_16_aad_None = r"""'2617EBCC'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_iv_16_aad_None = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_Array_iv_16_aad_None = r"""'69C5'""" + +example_mode_aes_192_cfb128_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_iv_16_aad_None = r"""'45450639'""" + +example_mode_aes_192_cfb128_datatype_IPv6_iv_16_aad_None = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_cfb128_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Enum16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_256_cfb128_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_UInt16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_UInt32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_UInt64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Int16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_Int32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_Int64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Float32_iv_16_aad_None = r"""'69580828'""" + +example_mode_aes_256_cfb128_datatype_Float64_iv_16_aad_None = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_iv_16_aad_None = r"""'49168817'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_iv_16_aad_None = r"""'491688173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_iv_16_aad_None = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_cfb128_datatype_UUID_iv_16_aad_None = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_cfb128_datatype_Date_iv_16_aad_None = r"""'3F1F'""" + +example_mode_aes_256_cfb128_datatype_DateTime_iv_16_aad_None = r"""'27888449'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_iv_16_aad_None = r"""'42E9857553DF4488'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_Array_iv_16_aad_None = r"""'685A'""" + +example_mode_aes_256_cfb128_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_iv_16_aad_None = r"""'44DA69BC'""" + +example_mode_aes_256_cfb128_datatype_IPv6_iv_16_aad_None = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_cfb128_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Enum16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_128_ofb_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_UInt16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_UInt32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_UInt64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Int16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_Int32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_Int64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Float32_iv_None_aad_None = r"""'10DF418A'""" + +example_mode_aes_128_ofb_datatype_Float64_iv_None_aad_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_ofb_datatype_Decimal32_iv_None_aad_None = r"""'3091C1B5'""" + +example_mode_aes_128_ofb_datatype_Decimal64_iv_None_aad_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Decimal128_iv_None_aad_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_ofb_datatype_UUID_iv_None_aad_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_ofb_datatype_Date_iv_None_aad_None = r"""'4698'""" + +example_mode_aes_128_ofb_datatype_DateTime_iv_None_aad_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_ofb_datatype_DateTime64_iv_None_aad_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_Array_iv_None_aad_None = r"""'11DD'""" + +example_mode_aes_128_ofb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_iv_None_aad_None = r"""'3D5D201E'""" + +example_mode_aes_128_ofb_datatype_IPv6_iv_None_aad_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_ofb_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Enum16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_192_ofb_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_UInt16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_UInt32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_UInt64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Int16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_Int32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_Int64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Float32_iv_None_aad_None = r"""'07B799A2'""" + +example_mode_aes_192_ofb_datatype_Float64_iv_None_aad_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_ofb_datatype_Decimal32_iv_None_aad_None = r"""'27F9199D'""" + +example_mode_aes_192_ofb_datatype_Decimal64_iv_None_aad_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Decimal128_iv_None_aad_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_ofb_datatype_UUID_iv_None_aad_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_ofb_datatype_Date_iv_None_aad_None = r"""'51F0'""" + +example_mode_aes_192_ofb_datatype_DateTime_iv_None_aad_None = r"""'496715C3'""" + +example_mode_aes_192_ofb_datatype_DateTime64_iv_None_aad_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_Array_iv_None_aad_None = r"""'06B5'""" + +example_mode_aes_192_ofb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_iv_None_aad_None = r"""'2A35F836'""" + +example_mode_aes_192_ofb_datatype_IPv6_iv_None_aad_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_ofb_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Enum16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_256_ofb_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_UInt16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_UInt32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_UInt64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Int16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_Int32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_Int64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Float32_iv_None_aad_None = r"""'B08E4FA1'""" + +example_mode_aes_256_ofb_datatype_Float64_iv_None_aad_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_ofb_datatype_Decimal32_iv_None_aad_None = r"""'90C0CF9E'""" + +example_mode_aes_256_ofb_datatype_Decimal64_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Decimal128_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_ofb_datatype_UUID_iv_None_aad_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_ofb_datatype_Date_iv_None_aad_None = r"""'E6C9'""" + +example_mode_aes_256_ofb_datatype_DateTime_iv_None_aad_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_ofb_datatype_DateTime64_iv_None_aad_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_Array_iv_None_aad_None = r"""'B18C'""" + +example_mode_aes_256_ofb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_iv_None_aad_None = r"""'9D0C2E35'""" + +example_mode_aes_256_ofb_datatype_IPv6_iv_None_aad_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_ofb_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Enum16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_128_ofb_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ofb_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ofb_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ofb_datatype_UInt16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_UInt32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_UInt64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Int16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_Int32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_Int64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Float32_iv_16_aad_None = r"""'328829BB'""" + +example_mode_aes_128_ofb_datatype_Float64_iv_16_aad_None = r"""'3288A984DD060F67'""" + +example_mode_aes_128_ofb_datatype_Decimal32_iv_16_aad_None = r"""'12C6A984'""" + +example_mode_aes_128_ofb_datatype_Decimal64_iv_16_aad_None = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Decimal128_iv_16_aad_None = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_ofb_datatype_UUID_iv_16_aad_None = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_ofb_datatype_Date_iv_16_aad_None = r"""'64CF'""" + +example_mode_aes_128_ofb_datatype_DateTime_iv_16_aad_None = r"""'7C58A5DA'""" + +example_mode_aes_128_ofb_datatype_DateTime64_iv_16_aad_None = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ofb_datatype_Array_iv_16_aad_None = r"""'338A'""" + +example_mode_aes_128_ofb_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_iv_16_aad_None = r"""'1F0A482F'""" + +example_mode_aes_128_ofb_datatype_IPv6_iv_16_aad_None = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_ofb_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Enum16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_192_ofb_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ofb_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ofb_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ofb_datatype_UInt16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_UInt32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_UInt64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Int16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_Int32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_Int64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Float32_iv_16_aad_None = r"""'68C767AD'""" + +example_mode_aes_192_ofb_datatype_Float64_iv_16_aad_None = r"""'68C7E792B710878E'""" + +example_mode_aes_192_ofb_datatype_Decimal32_iv_16_aad_None = r"""'4889E792'""" + +example_mode_aes_192_ofb_datatype_Decimal64_iv_16_aad_None = r"""'4889E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Decimal128_iv_16_aad_None = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_ofb_datatype_UUID_iv_16_aad_None = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_ofb_datatype_Date_iv_16_aad_None = r"""'3E80'""" + +example_mode_aes_192_ofb_datatype_DateTime_iv_16_aad_None = r"""'2617EBCC'""" + +example_mode_aes_192_ofb_datatype_DateTime64_iv_16_aad_None = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ofb_datatype_Array_iv_16_aad_None = r"""'69C5'""" + +example_mode_aes_192_ofb_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_iv_16_aad_None = r"""'45450639'""" + +example_mode_aes_192_ofb_datatype_IPv6_iv_16_aad_None = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_ofb_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Enum16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_256_ofb_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ofb_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ofb_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ofb_datatype_UInt16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_UInt32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_UInt64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Int16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_Int32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_Int64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Float32_iv_16_aad_None = r"""'69580828'""" + +example_mode_aes_256_ofb_datatype_Float64_iv_16_aad_None = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_ofb_datatype_Decimal32_iv_16_aad_None = r"""'49168817'""" + +example_mode_aes_256_ofb_datatype_Decimal64_iv_16_aad_None = r"""'491688173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Decimal128_iv_16_aad_None = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_ofb_datatype_UUID_iv_16_aad_None = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_ofb_datatype_Date_iv_16_aad_None = r"""'3F1F'""" + +example_mode_aes_256_ofb_datatype_DateTime_iv_16_aad_None = r"""'27888449'""" + +example_mode_aes_256_ofb_datatype_DateTime64_iv_16_aad_None = r"""'42E9857553DF4488'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ofb_datatype_Array_iv_16_aad_None = r"""'685A'""" + +example_mode_aes_256_ofb_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_iv_16_aad_None = r"""'44DA69BC'""" + +example_mode_aes_256_ofb_datatype_IPv6_iv_16_aad_None = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_ofb_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Enum16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_128_gcm_datatype_String_iv_12_aad_None = r"""'DC48B85D412AEF42C46DA18E25139D5D9D'""" + +example_mode_aes_128_gcm_datatype_FixedString_iv_12_aad_None = r"""'DC48B85D412AEF42C46DA18E25139D5D9D'""" + +example_mode_aes_128_gcm_datatype_UInt8_iv_12_aad_None = r"""'EC55C7254FC59C53DDE89F0AF034F08A19'""" + +example_mode_aes_128_gcm_datatype_UInt16_iv_12_aad_None = r"""'EC9644C4CEFA3BB71CD48C619B03BD238C18'""" + +example_mode_aes_128_gcm_datatype_UInt32_iv_12_aad_None = r"""'EC96C142DE5B4B621603E2ADD041DBAEB3F54F50'""" + +example_mode_aes_128_gcm_datatype_UInt64_iv_12_aad_None = r"""'EC96C142F4FF6F21DA0515DBB392E4C83187679A10F9D879'""" + +example_mode_aes_128_gcm_datatype_Int8_iv_12_aad_None = r"""'EC55C7254FC59C53DDE89F0AF034F08A19'""" + +example_mode_aes_128_gcm_datatype_Int16_iv_12_aad_None = r"""'EC9644C4CEFA3BB71CD48C619B03BD238C18'""" + +example_mode_aes_128_gcm_datatype_Int32_iv_12_aad_None = r"""'EC96C142DE5B4B621603E2ADD041DBAEB3F54F50'""" + +example_mode_aes_128_gcm_datatype_Int64_iv_12_aad_None = r"""'EC96C142F4FF6F21DA0515DBB392E4C83187679A10F9D879'""" + +example_mode_aes_128_gcm_datatype_Float32_iv_12_aad_None = r"""'ED96417D9A89D5DA7B2DA96F78D0D545BD724061'""" + +example_mode_aes_128_gcm_datatype_Float64_iv_12_aad_None = r"""'ED96C142F4FF9F1E36E4396725C81E2B56D2E0A42CB04CE9'""" + +example_mode_aes_128_gcm_datatype_Decimal32_iv_12_aad_None = r"""'CDD8C142960BD83213972E95DB26FFEC37308C56'""" + +example_mode_aes_128_gcm_datatype_Decimal64_iv_12_aad_None = r"""'CDD8C142F4FF6F219255868BB60628F03AE043D8943C1B7F'""" + +example_mode_aes_128_gcm_datatype_Decimal128_iv_12_aad_None = r"""'CDD8C142F4FF6F214AC2960E7EA36A6B1B1FC500659A9E81A843F13DCFC70751'""" + +example_mode_aes_128_gcm_datatype_UUID_iv_12_aad_None = r"""'0A87721EF03B9F40EA1945647E0511FB0552FE1E5264499B02AE50562A94AFF7'""" + +example_mode_aes_128_gcm_datatype_Date_iv_12_aad_None = r"""'BBD12669328AE5725A634A005E5B2C4BA877'""" + +example_mode_aes_128_gcm_datatype_DateTime_iv_12_aad_None = r"""'A346CD1C097FB5A48C84628E9C0C9F3BF4E3CBB2'""" + +example_mode_aes_128_gcm_datatype_DateTime64_iv_12_aad_None = r"""'C627CC209BFE6F21638D7F1465C11D12578CF38254273DC9'""" + +example_mode_aes_128_gcm_datatype_LowCardinality_iv_12_aad_None = r"""'DC48B85D412AEF42C46DA18E25139D5D9D'""" + +example_mode_aes_128_gcm_datatype_Array_iv_12_aad_None = r"""'EC94283AABA0C4FC46F56D71F9B3CB40288E'""" + +example_mode_aes_128_gcm_datatype_NULL_iv_12_aad_None = r"""'\\N'""" + +example_mode_aes_128_gcm_datatype_IPv4_iv_12_aad_None = r"""'C01420E9B1BBD6ED16DC1CC0ED7AE9773CD72297'""" + +example_mode_aes_128_gcm_datatype_IPv6_iv_12_aad_None = r"""'CD97CCFAF4FFEA824AC2960ED2BCEA6AC9427AAD6328E977BF356C8AEF93B556'""" + +example_mode_aes_128_gcm_datatype_Enum8_iv_12_aad_None = r"""'EC55C7254FC59C53DDE89F0AF034F08A19'""" + +example_mode_aes_128_gcm_datatype_Enum16_iv_12_aad_None = r"""'EC9644C4CEFA3BB71CD48C619B03BD238C18'""" + +example_mode_aes_192_gcm_datatype_String_iv_12_aad_None = r"""'7B34E3F4BAFCD2F3D493F843FFEBF9A415'""" + +example_mode_aes_192_gcm_datatype_FixedString_iv_12_aad_None = r"""'7B34E3F4BAFCD2F3D493F843FFEBF9A415'""" + +example_mode_aes_192_gcm_datatype_UInt8_iv_12_aad_None = r"""'4B5F8852220B8F5659FDE6C603D1C7E127'""" + +example_mode_aes_192_gcm_datatype_UInt16_iv_12_aad_None = r"""'4BFA728CD0043E08485EF39EEFEF8554BDD1'""" + +example_mode_aes_192_gcm_datatype_UInt32_iv_12_aad_None = r"""'4BFAA6C499D3637CF257889C690120FA80FBDBFD'""" + +example_mode_aes_192_gcm_datatype_UInt64_iv_12_aad_None = r"""'4BFAA6C467061FE32600E6AE21DD8A6D7AA0A38D2BE318EE'""" + +example_mode_aes_192_gcm_datatype_Int8_iv_12_aad_None = r"""'4B5F8852220B8F5659FDE6C603D1C7E127'""" + +example_mode_aes_192_gcm_datatype_Int16_iv_12_aad_None = r"""'4BFA728CD0043E08485EF39EEFEF8554BDD1'""" + +example_mode_aes_192_gcm_datatype_Int32_iv_12_aad_None = r"""'4BFAA6C499D3637CF257889C690120FA80FBDBFD'""" + +example_mode_aes_192_gcm_datatype_Int64_iv_12_aad_None = r"""'4BFAA6C467061FE32600E6AE21DD8A6D7AA0A38D2BE318EE'""" + +example_mode_aes_192_gcm_datatype_Float32_iv_12_aad_None = r"""'4AFA26FB812573A4DCCCFFB53F91E180A3F6EB95'""" + +example_mode_aes_192_gcm_datatype_Float64_iv_12_aad_None = r"""'4AFAA6C46706EFDC2968FA31495521544C0E792959D3DBA7'""" + +example_mode_aes_192_gcm_datatype_Decimal32_iv_12_aad_None = r"""'6AB4A6C4382BD5F8E4915BD5701D917BA8651BCA'""" + +example_mode_aes_192_gcm_datatype_Decimal64_iv_12_aad_None = r"""'6AB4A6C467061FE387F8502A371B592463BC120C037DD8D9'""" + +example_mode_aes_192_gcm_datatype_Decimal128_iv_12_aad_None = r"""'6AB4A6C467061FE343593C80B73515B94130370E40D09BDBE9EDB5BEAF29C541'""" + +example_mode_aes_192_gcm_datatype_UUID_iv_12_aad_None = r"""'ADEB159863C2EF82E382EFEAB7936E296A91D29B6A49269870CC866604C82451'""" + +example_mode_aes_192_gcm_datatype_Date_iv_12_aad_None = r"""'1CBD949DFC326906B13FCB565C1B28516877'""" + +example_mode_aes_192_gcm_datatype_DateTime_iv_12_aad_None = r"""'042AAA9AB3B5BBD234DF77B8C35BD4EBF561AEC0'""" + +example_mode_aes_192_gcm_datatype_DateTime64_iv_12_aad_None = r"""'614BABA608071FE304B3939091C0CFC91D1371D5FBD3F6BE'""" + +example_mode_aes_192_gcm_datatype_LowCardinality_iv_12_aad_None = r"""'7B34E3F4BAFCD2F3D493F843FFEBF9A415'""" + +example_mode_aes_192_gcm_datatype_Array_iv_12_aad_None = r"""'4BF887E38B2870FDEECD6305AEA010298096'""" + +example_mode_aes_192_gcm_datatype_NULL_iv_12_aad_None = r"""'\\N'""" + +example_mode_aes_192_gcm_datatype_IPv4_iv_12_aad_None = r"""'6778476F9B3687FD4D022727F4641E697D35D8D5'""" + +example_mode_aes_192_gcm_datatype_IPv6_iv_12_aad_None = r"""'6AFBAB7C67069A4043593C801B2A95B81CF6DF4E6D63A84A0DF7939A3FBE5B2D'""" + +example_mode_aes_192_gcm_datatype_Enum8_iv_12_aad_None = r"""'4B5F8852220B8F5659FDE6C603D1C7E127'""" + +example_mode_aes_192_gcm_datatype_Enum16_iv_12_aad_None = r"""'4BFA728CD0043E08485EF39EEFEF8554BDD1'""" + +example_mode_aes_256_gcm_datatype_String_iv_12_aad_None = r"""'67B83EFC31C169D7613D6881E954F624C2'""" + +example_mode_aes_256_gcm_datatype_FixedString_iv_12_aad_None = r"""'67B83EFC31C169D7613D6881E954F624C2'""" + +example_mode_aes_256_gcm_datatype_UInt8_iv_12_aad_None = r"""'574B67F06530273DD181658D384D73A2F5'""" + +example_mode_aes_256_gcm_datatype_UInt16_iv_12_aad_None = r"""'57CBC5BA7DF134BBAE0153E139EB9D7B8D18'""" + +example_mode_aes_256_gcm_datatype_UInt32_iv_12_aad_None = r"""'57CB6C3A286A4C6BDCEBF8ECEE345F12B5F44729'""" + +example_mode_aes_256_gcm_datatype_UInt64_iv_12_aad_None = r"""'57CB6C3AFC8427B9E3A61ABE0A227F1C2ADB2694F324393D'""" + +example_mode_aes_256_gcm_datatype_Int8_iv_12_aad_None = r"""'574B67F06530273DD181658D384D73A2F5'""" + +example_mode_aes_256_gcm_datatype_Int16_iv_12_aad_None = r"""'57CBC5BA7DF134BBAE0153E139EB9D7B8D18'""" + +example_mode_aes_256_gcm_datatype_Int32_iv_12_aad_None = r"""'57CB6C3A286A4C6BDCEBF8ECEE345F12B5F44729'""" + +example_mode_aes_256_gcm_datatype_Int64_iv_12_aad_None = r"""'57CB6C3AFC8427B9E3A61ABE0A227F1C2ADB2694F324393D'""" + +example_mode_aes_256_gcm_datatype_Float32_iv_12_aad_None = r"""'56CBEC055F35D9E71D5AA0252DB60716A99B1CA2'""" + +example_mode_aes_256_gcm_datatype_Float64_iv_12_aad_None = r"""'56CB6C3AFC84D786DF3A9796E0FDCECFCBBC26274A60F542'""" + +example_mode_aes_256_gcm_datatype_Decimal32_iv_12_aad_None = r"""'76856C3AE585D8FCE599B3C2E0F077FAD5FFF7D4'""" + +example_mode_aes_256_gcm_datatype_Decimal64_iv_12_aad_None = r"""'76856C3AFC8427B92E498E2933503432241F0E7C932F89C0'""" + +example_mode_aes_256_gcm_datatype_Decimal128_iv_12_aad_None = r"""'76856C3AFC8427B90E79078689172F14CAA5CA7BDC5F7FCD915CAE4EC302FF1C'""" + +example_mode_aes_256_gcm_datatype_UUID_iv_12_aad_None = r"""'B1DADF66F840D7D8AEA2D4EC89B1548437C22BB5B8D92A3A139ACED61DB3F2B3'""" + +example_mode_aes_256_gcm_datatype_Date_iv_12_aad_None = r"""'008CFC6AFC8484AADA010876EF4686BC3528'""" + +example_mode_aes_256_gcm_datatype_DateTime_iv_12_aad_None = r"""'181B606489B8E56FEB8D9950EAE27205E36C94FB'""" + +example_mode_aes_256_gcm_datatype_DateTime64_iv_12_aad_None = r"""'7D7A6158938527B9F938A05622B5957BD801AE303908AD22'""" + +example_mode_aes_256_gcm_datatype_LowCardinality_iv_12_aad_None = r"""'67B83EFC31C169D7613D6881E954F624C2'""" + +example_mode_aes_256_gcm_datatype_Array_iv_12_aad_None = r"""'57C908D31B11B2CE494A61EC468B149A9D77'""" + +example_mode_aes_256_gcm_datatype_NULL_iv_12_aad_None = r"""'\\N'""" + +example_mode_aes_256_gcm_datatype_IPv4_iv_12_aad_None = r"""'7B498D91B274D5D6748C43E4345552318BD05DA6'""" + +example_mode_aes_256_gcm_datatype_IPv6_iv_12_aad_None = r"""'76CA6182FC84A21A0E7907862508AF15CCE4089698265E4B0656598079CE56CC'""" + +example_mode_aes_256_gcm_datatype_Enum8_iv_12_aad_None = r"""'574B67F06530273DD181658D384D73A2F5'""" + +example_mode_aes_256_gcm_datatype_Enum16_iv_12_aad_None = r"""'57CBC5BA7DF134BBAE0153E139EB9D7B8D18'""" + +example_mode_aes_128_gcm_datatype_String_iv_12_aad_True = r"""'DCFAF1088D33EF99F1D06E3D14F265FD41'""" + +example_mode_aes_128_gcm_datatype_FixedString_iv_12_aad_True = r"""'DCFAF1088D33EF99F1D06E3D14F265FD41'""" + +example_mode_aes_128_gcm_datatype_UInt8_iv_12_aad_True = r"""'ECE78E7083DC9C88E85550B9C1D5082AC5'""" + +example_mode_aes_128_gcm_datatype_UInt16_iv_12_aad_True = r"""'EC96F68D9B3622B7C7E131AE28325CDB2CC4'""" + +example_mode_aes_128_gcm_datatype_UInt32_iv_12_aad_True = r"""'EC96C1426C121EAE0F0339986D8E689F520DEF8C'""" + +example_mode_aes_128_gcm_datatype_UInt64_iv_12_aad_True = r"""'EC96C142F4FF6F21684C4017AA923FFD8C48D4ABF10178A5'""" + +example_mode_aes_128_gcm_datatype_Int8_iv_12_aad_True = r"""'ECE78E7083DC9C88E85550B9C1D5082AC5'""" + +example_mode_aes_128_gcm_datatype_Int16_iv_12_aad_True = r"""'EC96F68D9B3622B7C7E131AE28325CDB2CC4'""" + +example_mode_aes_128_gcm_datatype_Int32_iv_12_aad_True = r"""'EC96C1426C121EAE0F0339986D8E689F520DEF8C'""" + +example_mode_aes_128_gcm_datatype_Int64_iv_12_aad_True = r"""'EC96C142F4FF6F21684C4017AA923FFD8C48D4ABF10178A5'""" + +example_mode_aes_128_gcm_datatype_Float32_iv_12_aad_True = r"""'ED96417D28C08016622D725AC51F66745C8AE0BD'""" + +example_mode_aes_128_gcm_datatype_Float64_iv_12_aad_True = r"""'ED96C142F4FF9F1E84AD6CAB3CC8C51EEB1D5395CD48EC35'""" + +example_mode_aes_128_gcm_datatype_Decimal32_iv_12_aad_True = r"""'CDD8C14224428DFE0A97F5A066E94CDDD6C82C8A'""" + +example_mode_aes_128_gcm_datatype_Decimal64_iv_12_aad_True = r"""'CDD8C142F4FF6F21201CD347AF06F3C5872FF0E975C4BBA3'""" + +example_mode_aes_128_gcm_datatype_Decimal128_iv_12_aad_True = r"""'CDD8C142F4FF6F214AC2960E7EA36A6BA95690CC7C9A45B4158C420C2E3FA78D'""" + +example_mode_aes_128_gcm_datatype_UUID_iv_12_aad_True = r"""'0A87721EF03B9F40EA1945647E0511FBB71BABD24B6492AEBF61E367CB6C0F2B'""" + +example_mode_aes_128_gcm_datatype_Date_iv_12_aad_True = r"""'BBD194206746FC728156F7CFED6ACDB308AB'""" + +example_mode_aes_128_gcm_datatype_DateTime_iv_12_aad_True = r"""'A346CD1CBB36E0689584B9BB21C32C0A151B6B6E'""" + +example_mode_aes_128_gcm_datatype_DateTime64_iv_12_aad_True = r"""'C627CC209BFE6F21D1C42AD87CC1C627EA4340B3B5DF9D15'""" + +example_mode_aes_128_gcm_datatype_LowCardinality_iv_12_aad_True = r"""'DCFAF1088D33EF99F1D06E3D14F265FD41'""" + +example_mode_aes_128_gcm_datatype_Array_iv_12_aad_True = r"""'EC949A73FE6CDDFC9DC0D0BE4A822AB88852'""" + +example_mode_aes_128_gcm_datatype_NULL_iv_12_aad_True = r"""'\\N'""" + +example_mode_aes_128_gcm_datatype_IPv4_iv_12_aad_True = r"""'C01420E903F283210FDCC7F550B55A46DD2F824B'""" + +example_mode_aes_128_gcm_datatype_IPv6_iv_12_aad_True = r"""'CD97CCFAF4FFEA824AC2960ED2BCEA6A7B0B2F617A28324202FADFBB0E6B158A'""" + +example_mode_aes_128_gcm_datatype_Enum8_iv_12_aad_True = r"""'ECE78E7083DC9C88E85550B9C1D5082AC5'""" + +example_mode_aes_128_gcm_datatype_Enum16_iv_12_aad_True = r"""'EC96F68D9B3622B7C7E131AE28325CDB2CC4'""" + +example_mode_aes_192_gcm_datatype_String_iv_12_aad_True = r"""'7BB72D91D66E3C93B34FBAFF92526E1A0E'""" + +example_mode_aes_192_gcm_datatype_FixedString_iv_12_aad_True = r"""'7BB72D91D66E3C93B34FBAFF92526E1A0E'""" + +example_mode_aes_192_gcm_datatype_UInt8_iv_12_aad_True = r"""'4BDC46374E9961363E21A47A6E68505F3C'""" + +example_mode_aes_192_gcm_datatype_UInt16_iv_12_aad_True = r"""'4BFAF142B568ACE628392FDC53823CC303CA'""" + +example_mode_aes_192_gcm_datatype_UInt32_iv_12_aad_True = r"""'4BFAA6C41A1D061060B9E8FBB5439C97396C65E6'""" + +example_mode_aes_192_gcm_datatype_UInt64_iv_12_aad_True = r"""'4BFAA6C467061FE3A5CE83C2B333EA0AA6E21FE09274A6F5'""" + +example_mode_aes_192_gcm_datatype_Int8_iv_12_aad_True = r"""'4BDC46374E9961363E21A47A6E68505F3C'""" + +example_mode_aes_192_gcm_datatype_Int16_iv_12_aad_True = r"""'4BFAF142B568ACE628392FDC53823CC303CA'""" + +example_mode_aes_192_gcm_datatype_Int32_iv_12_aad_True = r"""'4BFAA6C41A1D061060B9E8FBB5439C97396C65E6'""" + +example_mode_aes_192_gcm_datatype_Int64_iv_12_aad_True = r"""'4BFAA6C467061FE3A5CE83C2B333EA0AA6E21FE09274A6F5'""" + +example_mode_aes_192_gcm_datatype_Float32_iv_12_aad_True = r"""'4AFA26FB02EB16C84E229FD2E3D35DED1A61558E'""" + +example_mode_aes_192_gcm_datatype_Float64_iv_12_aad_True = r"""'4AFAA6C46706EFDCAAA69F5DDBBB4133904CC544E04465BC'""" + +example_mode_aes_192_gcm_datatype_Decimal32_iv_12_aad_True = r"""'6AB4A6C4BBE5B094767F3BB2AC5F2D1611F2A5D1'""" + +example_mode_aes_192_gcm_datatype_Decimal64_iv_12_aad_True = r"""'6AB4A6C467061FE304363546A5F53943BFFEAE61BAEA66C2'""" + +example_mode_aes_192_gcm_datatype_Decimal128_iv_12_aad_True = r"""'6AB4A6C467061FE343593C80B73515B9C2FE5262D23EFBBC35AF09D316BE7B5A'""" + +example_mode_aes_192_gcm_datatype_UUID_iv_12_aad_True = r"""'ADEB159863C2EF82E382EFEAB7936E29E95FB7F7F8A746FFAC8E3A0BBD5F9A4A'""" + +example_mode_aes_192_gcm_datatype_Date_iv_12_aad_True = r"""'1CBD1753995EFBE8D1581714E07691C6D66C'""" + +example_mode_aes_192_gcm_datatype_DateTime_iv_12_aad_True = r"""'042AAA9A307BDEBEA63117DF1F1968864CF610DB'""" + +example_mode_aes_192_gcm_datatype_DateTime64_iv_12_aad_True = r"""'614BABA608071FE3877DF6FC032EAFAEC151CDB8424448A5'""" + +example_mode_aes_192_gcm_datatype_LowCardinality_iv_12_aad_True = r"""'7BB72D91D66E3C93B34FBAFF92526E1A0E'""" + +example_mode_aes_192_gcm_datatype_Array_iv_12_aad_True = r"""'4BF8042DEE44E2138EAABF4712CDA9BE3E8D'""" + +example_mode_aes_192_gcm_datatype_NULL_iv_12_aad_True = r"""'\\N'""" + +example_mode_aes_192_gcm_datatype_IPv4_iv_12_aad_True = r"""'6778476F18F8E291DFEC47402826A204C4A266CE'""" + +example_mode_aes_192_gcm_datatype_IPv6_iv_12_aad_True = r"""'6AFBAB7C67069A4043593C801B2A95B89F38BA22FF8DC82DD1B52FF78629E536'""" + +example_mode_aes_192_gcm_datatype_Enum8_iv_12_aad_True = r"""'4BDC46374E9961363E21A47A6E68505F3C'""" + +example_mode_aes_192_gcm_datatype_Enum16_iv_12_aad_True = r"""'4BFAF142B568ACE628392FDC53823CC303CA'""" + +example_mode_aes_256_gcm_datatype_String_iv_12_aad_True = r"""'672AAAA73DCD5DEBC924C34E8F6E2678F8'""" + +example_mode_aes_256_gcm_datatype_FixedString_iv_12_aad_True = r"""'672AAAA73DCD5DEBC924C34E8F6E2678F8'""" + +example_mode_aes_256_gcm_datatype_UInt8_iv_12_aad_True = r"""'57D9F3AB693C13017998CE425E77A3FECF'""" + +example_mode_aes_256_gcm_datatype_UInt16_iv_12_aad_True = r"""'57CB572E26FD388F92A94A4AF68DA7ABD122'""" + +example_mode_aes_256_gcm_datatype_UInt32_iv_12_aad_True = r"""'57CB6C3ABAFE1767D0DFC444F79F90748F241B13'""" + +example_mode_aes_256_gcm_datatype_UInt64_iv_12_aad_True = r"""'57CB6C3AFC8427B9713241B2061643B43370E9F2C9F46507'""" + +example_mode_aes_256_gcm_datatype_Int8_iv_12_aad_True = r"""'57D9F3AB693C13017998CE425E77A3FECF'""" + +example_mode_aes_256_gcm_datatype_Int16_iv_12_aad_True = r"""'57CB572E26FD388F92A94A4AF68DA7ABD122'""" + +example_mode_aes_256_gcm_datatype_Int32_iv_12_aad_True = r"""'57CB6C3ABAFE1767D0DFC444F79F90748F241B13'""" + +example_mode_aes_256_gcm_datatype_Int64_iv_12_aad_True = r"""'57CB6C3AFC8427B9713241B2061643B43370E9F2C9F46507'""" + +example_mode_aes_256_gcm_datatype_Float32_iv_12_aad_True = r"""'56CBEC05CDA182EB116E9C8D341DC870934B4098'""" + +example_mode_aes_256_gcm_datatype_Float64_iv_12_aad_True = r"""'56CB6C3AFC84D7864DAECC9AECC9F267D217E94170B0A978'""" + +example_mode_aes_256_gcm_datatype_Decimal32_iv_12_aad_True = r"""'76856C3A771183F0E9AD8F6AF95BB89CEF2FABEE'""" + +example_mode_aes_256_gcm_datatype_Decimal64_iv_12_aad_True = r"""'76856C3AFC8427B9BCDDD5253F64089A3DB4C11AA9FFD5FA'""" + +example_mode_aes_256_gcm_datatype_Decimal128_iv_12_aad_True = r"""'76856C3AFC8427B90E79078689172F1458319177D06B436588F76128F9D2A326'""" + +example_mode_aes_256_gcm_datatype_UUID_iv_12_aad_True = r"""'B1DADF66F840D7D8AEA2D4EC89B15484A55670B9B4ED16920A3101B02763AE89'""" + +example_mode_aes_256_gcm_datatype_Date_iv_12_aad_True = r"""'008C6EFEA788889EE6A911DD2020BC6C6912'""" + +example_mode_aes_256_gcm_datatype_DateTime_iv_12_aad_True = r"""'181B60641B2CBE63E7B9A5F8F349BD63D9BCC8C1'""" + +example_mode_aes_256_gcm_datatype_DateTime64_iv_12_aad_True = r"""'7D7A6158938527B96BACFB5A2E81A9D3C1AA615603D8F118'""" + +example_mode_aes_256_gcm_datatype_LowCardinality_iv_12_aad_True = r"""'672AAAA73DCD5DEBC924C34E8F6E2678F8'""" + +example_mode_aes_256_gcm_datatype_Array_iv_12_aad_True = r"""'57C99A47401DBEFA75E2784789ED2E4AC14D'""" + +example_mode_aes_256_gcm_datatype_NULL_iv_12_aad_True = r"""'\\N'""" + +example_mode_aes_256_gcm_datatype_IPv4_iv_12_aad_True = r"""'7B498D9120E08EDA78B87F4C2DFE9D57B100019C'""" + +example_mode_aes_256_gcm_datatype_IPv6_iv_12_aad_True = r"""'76CA6182FC84A21A0E7907862508AF155E70539A941262E31FFD96E6431E0AF6'""" + +example_mode_aes_256_gcm_datatype_Enum8_iv_12_aad_True = r"""'57D9F3AB693C13017998CE425E77A3FECF'""" + +example_mode_aes_256_gcm_datatype_Enum16_iv_12_aad_True = r"""'57CB572E26FD388F92A94A4AF68DA7ABD122'""" + +example_mode_aes_128_ctr_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ctr_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ctr_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ctr_datatype_UInt16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ctr_datatype_UInt32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ctr_datatype_UInt64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ctr_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ctr_datatype_Int16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ctr_datatype_Int32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ctr_datatype_Int64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ctr_datatype_Float32_iv_None_aad_None = r"""'10DF418A'""" + +example_mode_aes_128_ctr_datatype_Float64_iv_None_aad_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_ctr_datatype_Decimal32_iv_None_aad_None = r"""'3091C1B5'""" + +example_mode_aes_128_ctr_datatype_Decimal64_iv_None_aad_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_ctr_datatype_Decimal128_iv_None_aad_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_ctr_datatype_UUID_iv_None_aad_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_ctr_datatype_Date_iv_None_aad_None = r"""'4698'""" + +example_mode_aes_128_ctr_datatype_DateTime_iv_None_aad_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_ctr_datatype_DateTime64_iv_None_aad_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_ctr_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ctr_datatype_Array_iv_None_aad_None = r"""'11DD'""" + +example_mode_aes_128_ctr_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_ctr_datatype_IPv4_iv_None_aad_None = r"""'3D5D201E'""" + +example_mode_aes_128_ctr_datatype_IPv6_iv_None_aad_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_ctr_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ctr_datatype_Enum16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_192_ctr_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ctr_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ctr_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ctr_datatype_UInt16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ctr_datatype_UInt32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ctr_datatype_UInt64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ctr_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ctr_datatype_Int16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ctr_datatype_Int32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ctr_datatype_Int64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ctr_datatype_Float32_iv_None_aad_None = r"""'07B799A2'""" + +example_mode_aes_192_ctr_datatype_Float64_iv_None_aad_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_ctr_datatype_Decimal32_iv_None_aad_None = r"""'27F9199D'""" + +example_mode_aes_192_ctr_datatype_Decimal64_iv_None_aad_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_ctr_datatype_Decimal128_iv_None_aad_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_ctr_datatype_UUID_iv_None_aad_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_ctr_datatype_Date_iv_None_aad_None = r"""'51F0'""" + +example_mode_aes_192_ctr_datatype_DateTime_iv_None_aad_None = r"""'496715C3'""" + +example_mode_aes_192_ctr_datatype_DateTime64_iv_None_aad_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_ctr_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ctr_datatype_Array_iv_None_aad_None = r"""'06B5'""" + +example_mode_aes_192_ctr_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_ctr_datatype_IPv4_iv_None_aad_None = r"""'2A35F836'""" + +example_mode_aes_192_ctr_datatype_IPv6_iv_None_aad_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_ctr_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ctr_datatype_Enum16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_256_ctr_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ctr_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ctr_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ctr_datatype_UInt16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ctr_datatype_UInt32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ctr_datatype_UInt64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ctr_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ctr_datatype_Int16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ctr_datatype_Int32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ctr_datatype_Int64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ctr_datatype_Float32_iv_None_aad_None = r"""'B08E4FA1'""" + +example_mode_aes_256_ctr_datatype_Float64_iv_None_aad_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_ctr_datatype_Decimal32_iv_None_aad_None = r"""'90C0CF9E'""" + +example_mode_aes_256_ctr_datatype_Decimal64_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_ctr_datatype_Decimal128_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_ctr_datatype_UUID_iv_None_aad_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_ctr_datatype_Date_iv_None_aad_None = r"""'E6C9'""" + +example_mode_aes_256_ctr_datatype_DateTime_iv_None_aad_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_ctr_datatype_DateTime64_iv_None_aad_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_ctr_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ctr_datatype_Array_iv_None_aad_None = r"""'B18C'""" + +example_mode_aes_256_ctr_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_ctr_datatype_IPv4_iv_None_aad_None = r"""'9D0C2E35'""" + +example_mode_aes_256_ctr_datatype_IPv6_iv_None_aad_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_ctr_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ctr_datatype_Enum16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_128_ctr_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ctr_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ctr_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ctr_datatype_UInt16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ctr_datatype_UInt32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ctr_datatype_UInt64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ctr_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ctr_datatype_Int16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ctr_datatype_Int32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ctr_datatype_Int64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ctr_datatype_Float32_iv_16_aad_None = r"""'328829BB'""" + +example_mode_aes_128_ctr_datatype_Float64_iv_16_aad_None = r"""'3288A984DD060F67'""" + +example_mode_aes_128_ctr_datatype_Decimal32_iv_16_aad_None = r"""'12C6A984'""" + +example_mode_aes_128_ctr_datatype_Decimal64_iv_16_aad_None = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_ctr_datatype_Decimal128_iv_16_aad_None = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_ctr_datatype_UUID_iv_16_aad_None = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_ctr_datatype_Date_iv_16_aad_None = r"""'64CF'""" + +example_mode_aes_128_ctr_datatype_DateTime_iv_16_aad_None = r"""'7C58A5DA'""" + +example_mode_aes_128_ctr_datatype_DateTime64_iv_16_aad_None = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_ctr_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ctr_datatype_Array_iv_16_aad_None = r"""'338A'""" + +example_mode_aes_128_ctr_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_ctr_datatype_IPv4_iv_16_aad_None = r"""'1F0A482F'""" + +example_mode_aes_128_ctr_datatype_IPv6_iv_16_aad_None = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_ctr_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ctr_datatype_Enum16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_192_ctr_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ctr_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ctr_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ctr_datatype_UInt16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ctr_datatype_UInt32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ctr_datatype_UInt64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ctr_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ctr_datatype_Int16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ctr_datatype_Int32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ctr_datatype_Int64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ctr_datatype_Float32_iv_16_aad_None = r"""'68C767AD'""" + +example_mode_aes_192_ctr_datatype_Float64_iv_16_aad_None = r"""'68C7E792B710878E'""" + +example_mode_aes_192_ctr_datatype_Decimal32_iv_16_aad_None = r"""'4889E792'""" + +example_mode_aes_192_ctr_datatype_Decimal64_iv_16_aad_None = r"""'4889E792B71077B1'""" + +example_mode_aes_192_ctr_datatype_Decimal128_iv_16_aad_None = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_ctr_datatype_UUID_iv_16_aad_None = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_ctr_datatype_Date_iv_16_aad_None = r"""'3E80'""" + +example_mode_aes_192_ctr_datatype_DateTime_iv_16_aad_None = r"""'2617EBCC'""" + +example_mode_aes_192_ctr_datatype_DateTime64_iv_16_aad_None = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_ctr_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ctr_datatype_Array_iv_16_aad_None = r"""'69C5'""" + +example_mode_aes_192_ctr_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_ctr_datatype_IPv4_iv_16_aad_None = r"""'45450639'""" + +example_mode_aes_192_ctr_datatype_IPv6_iv_16_aad_None = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_ctr_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ctr_datatype_Enum16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_256_ctr_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ctr_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ctr_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ctr_datatype_UInt16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ctr_datatype_UInt32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ctr_datatype_UInt64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ctr_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ctr_datatype_Int16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ctr_datatype_Int32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ctr_datatype_Int64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ctr_datatype_Float32_iv_16_aad_None = r"""'69580828'""" + +example_mode_aes_256_ctr_datatype_Float64_iv_16_aad_None = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_ctr_datatype_Decimal32_iv_16_aad_None = r"""'49168817'""" + +example_mode_aes_256_ctr_datatype_Decimal64_iv_16_aad_None = r"""'491688173CDE4488'""" + +example_mode_aes_256_ctr_datatype_Decimal128_iv_16_aad_None = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_ctr_datatype_UUID_iv_16_aad_None = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_ctr_datatype_Date_iv_16_aad_None = r"""'3F1F'""" + +example_mode_aes_256_ctr_datatype_DateTime_iv_16_aad_None = r"""'27888449'""" + +example_mode_aes_256_ctr_datatype_DateTime64_iv_16_aad_None = r"""'42E9857553DF4488'""" + +example_mode_aes_256_ctr_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ctr_datatype_Array_iv_16_aad_None = r"""'685A'""" + +example_mode_aes_256_ctr_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_ctr_datatype_IPv4_iv_16_aad_None = r"""'44DA69BC'""" + +example_mode_aes_256_ctr_datatype_IPv6_iv_16_aad_None = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_ctr_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ctr_datatype_Enum16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_128_ecb_datatype_bytes_iv_None_aad_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_ecb_datatype_emptystring_iv_None_aad_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_192_ecb_datatype_bytes_iv_None_aad_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_ecb_datatype_emptystring_iv_None_aad_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_256_ecb_datatype_bytes_iv_None_aad_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_ecb_datatype_emptystring_iv_None_aad_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_128_cbc_datatype_bytes_iv_None_aad_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_cbc_datatype_emptystring_iv_None_aad_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_192_cbc_datatype_bytes_iv_None_aad_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_cbc_datatype_emptystring_iv_None_aad_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_256_cbc_datatype_bytes_iv_None_aad_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_cbc_datatype_emptystring_iv_None_aad_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_128_cbc_datatype_bytes_iv_16_aad_None = r"""'CDA4B7027137998B9A33C2096C9A50DD'""" + +example_mode_aes_128_cbc_datatype_emptystring_iv_16_aad_None = r"""'56A77308430BA344FFBF016999795ED5'""" + +example_mode_aes_192_cbc_datatype_bytes_iv_16_aad_None = r"""'67771349942D4F812553F2E1E3FFB276'""" + +example_mode_aes_192_cbc_datatype_emptystring_iv_16_aad_None = r"""'62E9214DB5E239F0CAD31ADF26AB313F'""" + +example_mode_aes_256_cbc_datatype_bytes_iv_16_aad_None = r"""'6046ECF8094941C6DEC9278FF6F137E9'""" + +example_mode_aes_256_cbc_datatype_emptystring_iv_16_aad_None = r"""'4EC7785DA650D55B71B52816B1DB5AD3'""" + +example_mode_aes_128_cfb1_datatype_bytes_iv_None_aad_None = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_cfb1_datatype_bytes_iv_None_aad_None = r"""'00'""" + +example_mode_aes_192_cfb1_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_cfb1_datatype_bytes_iv_None_aad_None = r"""'B8'""" + +example_mode_aes_256_cfb1_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_cfb1_datatype_bytes_iv_16_aad_None = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_cfb1_datatype_bytes_iv_16_aad_None = r"""'07'""" + +example_mode_aes_192_cfb1_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_cfb1_datatype_bytes_iv_16_aad_None = r"""'7F'""" + +example_mode_aes_256_cfb1_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_cfb8_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_cfb8_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_cfb8_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_cfb8_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_cfb8_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_cfb8_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_cfb8_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_cfb8_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_cfb8_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_cfb8_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_cfb8_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_cfb8_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_cfb128_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_cfb128_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_cfb128_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_cfb128_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_cfb128_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_cfb128_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_cfb128_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_cfb128_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_cfb128_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_cfb128_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_cfb128_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_cfb128_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_ofb_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_ofb_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_ofb_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_ofb_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_ofb_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_ofb_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_ofb_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_ofb_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_ofb_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_ofb_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_ofb_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_ofb_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_gcm_datatype_bytes_iv_12_aad_None = r"""'ED7AF588306031432D60AE52CB0522C131'""" + +example_mode_aes_128_gcm_datatype_emptystring_iv_12_aad_None = r"""'BF165C92C4F89A189DCD3A9CCD549D64'""" + +example_mode_aes_192_gcm_datatype_bytes_iv_12_aad_None = r"""'4AF425C405715C1F91B04661C96F5942E9'""" + +example_mode_aes_192_gcm_datatype_emptystring_iv_12_aad_None = r"""'F95400D38FC8B708F21C1A5CC97E2611'""" + +example_mode_aes_256_gcm_datatype_bytes_iv_12_aad_None = r"""'56C3D480260AD4984887DA3D7CBDFB952B'""" + +example_mode_aes_256_gcm_datatype_emptystring_iv_12_aad_None = r"""'4C24D7C3019165A77A8AE2FB9E130FBB'""" + +example_mode_aes_128_gcm_datatype_bytes_iv_12_aad_True = r"""'EDC8BCDDFC79319818DD61E1FAE4DA61ED'""" + +example_mode_aes_128_gcm_datatype_emptystring_iv_12_aad_True = r"""'9E4606A8AD25466858006BD90DA530F5'""" + +example_mode_aes_192_gcm_datatype_bytes_iv_12_aad_True = r"""'4A77EBA169E3B27FF66C04DDA4D6CEFCF2'""" + +example_mode_aes_192_gcm_datatype_emptystring_iv_12_aad_True = r"""'43B2949BA8DFB2A1511C294394E0DFB4'""" + +example_mode_aes_256_gcm_datatype_bytes_iv_12_aad_True = r"""'565140DB2A06E0A4E09E71F21A872BC911'""" + +example_mode_aes_256_gcm_datatype_emptystring_iv_12_aad_True = r"""'555A1764B65804E27174F272CF88FC11'""" + +example_mode_aes_128_ctr_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_ctr_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_ctr_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_ctr_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_ctr_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_ctr_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_ctr_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_ctr_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_ctr_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_ctr_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_ctr_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_ctr_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_ecb_datatype_utf8string_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_128_ecb_datatype_utf8fixedstring_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_192_ecb_datatype_utf8string_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_192_ecb_datatype_utf8fixedstring_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_256_ecb_datatype_utf8string_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_256_ecb_datatype_utf8fixedstring_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_128_cbc_datatype_utf8string_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_192_cbc_datatype_utf8string_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_256_cbc_datatype_utf8string_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_128_cbc_datatype_utf8string_iv_16_aad_None = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_iv_16_aad_None = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_192_cbc_datatype_utf8string_iv_16_aad_None = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_iv_16_aad_None = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_256_cbc_datatype_utf8string_iv_16_aad_None = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_iv_16_aad_None = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_128_cfb1_datatype_utf8string_iv_None_aad_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_iv_None_aad_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_192_cfb1_datatype_utf8string_iv_None_aad_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_iv_None_aad_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_256_cfb1_datatype_utf8string_iv_None_aad_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_iv_None_aad_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_128_cfb1_datatype_utf8string_iv_16_aad_None = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_iv_16_aad_None = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_192_cfb1_datatype_utf8string_iv_16_aad_None = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_iv_16_aad_None = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_256_cfb1_datatype_utf8string_iv_16_aad_None = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_128_cfb8_datatype_utf8string_iv_None_aad_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_iv_None_aad_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_192_cfb8_datatype_utf8string_iv_None_aad_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_iv_None_aad_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_256_cfb8_datatype_utf8string_iv_None_aad_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_iv_None_aad_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_128_cfb8_datatype_utf8string_iv_16_aad_None = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_iv_16_aad_None = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_192_cfb8_datatype_utf8string_iv_16_aad_None = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_256_cfb8_datatype_utf8string_iv_16_aad_None = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_128_cfb128_datatype_utf8string_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_192_cfb128_datatype_utf8string_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_256_cfb128_datatype_utf8string_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_128_cfb128_datatype_utf8string_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_192_cfb128_datatype_utf8string_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_256_cfb128_datatype_utf8string_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_128_ofb_datatype_utf8string_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_192_ofb_datatype_utf8string_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_256_ofb_datatype_utf8string_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_128_ofb_datatype_utf8string_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_192_ofb_datatype_utf8string_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_256_ofb_datatype_utf8string_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_128_gcm_datatype_utf8string_iv_12_aad_None = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CDDC3AD7EBB18C53625907FE8745D8FFA'""" + +example_mode_aes_128_gcm_datatype_utf8fixedstring_iv_12_aad_None = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CDDC3AD7EBB18C53625907FE8745D8FFA'""" + +example_mode_aes_192_gcm_datatype_utf8string_iv_12_aad_None = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20D0A9639BEF09294270D1352D22DB2CAF'""" + +example_mode_aes_192_gcm_datatype_utf8fixedstring_iv_12_aad_None = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20D0A9639BEF09294270D1352D22DB2CAF'""" + +example_mode_aes_256_gcm_datatype_utf8string_iv_12_aad_None = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FCAC905BDB791BC316A9A7D21FB34951F0'""" + +example_mode_aes_256_gcm_datatype_utf8fixedstring_iv_12_aad_None = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FCAC905BDB791BC316A9A7D21FB34951F0'""" + +example_mode_aes_128_gcm_datatype_utf8string_iv_12_aad_True = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CD181E1471DC596491F22269EF75F4E5F'""" + +example_mode_aes_128_gcm_datatype_utf8fixedstring_iv_12_aad_True = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CD181E1471DC596491F22269EF75F4E5F'""" + +example_mode_aes_192_gcm_datatype_utf8string_iv_12_aad_True = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20936BD43E66BAD9322C30B6E5D64110BB'""" + +example_mode_aes_192_gcm_datatype_utf8fixedstring_iv_12_aad_True = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20936BD43E66BAD9322C30B6E5D64110BB'""" + +example_mode_aes_256_gcm_datatype_utf8string_iv_12_aad_True = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FC92F054FDB351668A3790BE74D20869DD'""" + +example_mode_aes_256_gcm_datatype_utf8fixedstring_iv_12_aad_True = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FC92F054FDB351668A3790BE74D20869DD'""" + +example_mode_aes_128_ctr_datatype_utf8string_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B60215F9423235540'""" + +example_mode_aes_128_ctr_datatype_utf8fixedstring_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B60215F9423235540'""" + +example_mode_aes_192_ctr_datatype_utf8string_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9EE89664A09F10327'""" + +example_mode_aes_192_ctr_datatype_utf8fixedstring_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9EE89664A09F10327'""" + +example_mode_aes_256_ctr_datatype_utf8string_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06519900DEC1DB5CF5'""" + +example_mode_aes_256_ctr_datatype_utf8fixedstring_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06519900DEC1DB5CF5'""" + +example_mode_aes_128_ctr_datatype_utf8string_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB437EE2D5260EB4958'""" + +example_mode_aes_128_ctr_datatype_utf8fixedstring_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB437EE2D5260EB4958'""" + +example_mode_aes_192_ctr_datatype_utf8string_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F039E1E85A0813F1D'""" + +example_mode_aes_192_ctr_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F039E1E85A0813F1D'""" + +example_mode_aes_256_ctr_datatype_utf8string_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C944F1E670DF94BAA'""" + +example_mode_aes_256_ctr_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C944F1E670DF94BAA'""" + diff --git a/tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot b/tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot new file mode 100644 index 00000000000..280dbf35be2 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot @@ -0,0 +1,3060 @@ +example_mode_aes_128_ecb_datatype_bytes_key_16_iv_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_ecb_datatype_emptystring_key_16_iv_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_utf8string_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_128_ecb_datatype_utf8fixedstring_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_128_ecb_datatype_String_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_FixedString_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_UInt8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_UInt16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_UInt32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_UInt64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Int8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Int16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_Int32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_Int64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Float32_key_16_iv_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_ecb_datatype_Float64_key_16_iv_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_ecb_datatype_Decimal32_key_16_iv_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_ecb_datatype_Decimal64_key_16_iv_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_ecb_datatype_Decimal128_key_16_iv_None = r"""'5F3DBFA74809E45E03980357B26787AFF30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_UUID_key_16_iv_None = r"""'FF9161B222B4A67481271035745F06D9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Date_key_16_iv_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_ecb_datatype_DateTime_key_16_iv_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_ecb_datatype_DateTime64_key_16_iv_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_ecb_datatype_LowCardinality_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_Array_key_16_iv_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_ecb_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_ecb_datatype_IPv4_key_16_iv_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_ecb_datatype_IPv6_key_16_iv_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Enum8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Enum16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_bytes_key_24_iv_None = r"""'0E7A1A2ED5C8BCC0B811B44D7FEA9E51'""" + +example_mode_aes_128_ecb_datatype_emptystring_key_24_iv_None = r"""'644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_utf8string_key_24_iv_None = r"""'4A970096F9F40F5507A678BF07F22B6B0644E044B974D161C223EA94DCB43B5D'""" + +example_mode_aes_128_ecb_datatype_utf8fixedstring_key_24_iv_None = r"""'4A970096F9F40F5507A678BF07F22B6B0644E044B974D161C223EA94DCB43B5D'""" + +example_mode_aes_128_ecb_datatype_String_key_24_iv_None = r"""'697175286BC73A26C572DBD9480738F3'""" + +example_mode_aes_128_ecb_datatype_FixedString_key_24_iv_None = r"""'697175286BC73A26C572DBD9480738F3'""" + +example_mode_aes_128_ecb_datatype_UInt8_key_24_iv_None = r"""'113A22E44AC1C4B397CC8204C069F5F4'""" + +example_mode_aes_128_ecb_datatype_UInt16_key_24_iv_None = r"""'94DD57978311932F2E9FED922796A023'""" + +example_mode_aes_128_ecb_datatype_UInt32_key_24_iv_None = r"""'CB84F00C70A72890AF6F7106AE8109CB'""" + +example_mode_aes_128_ecb_datatype_UInt64_key_24_iv_None = r"""'973944561BDA0D954449BEBD64C9ED7A'""" + +example_mode_aes_128_ecb_datatype_Int8_key_24_iv_None = r"""'113A22E44AC1C4B397CC8204C069F5F4'""" + +example_mode_aes_128_ecb_datatype_Int16_key_24_iv_None = r"""'94DD57978311932F2E9FED922796A023'""" + +example_mode_aes_128_ecb_datatype_Int32_key_24_iv_None = r"""'CB84F00C70A72890AF6F7106AE8109CB'""" + +example_mode_aes_128_ecb_datatype_Int64_key_24_iv_None = r"""'973944561BDA0D954449BEBD64C9ED7A'""" + +example_mode_aes_128_ecb_datatype_Float32_key_24_iv_None = r"""'7BCD3C1EB87CDA7BBA4B19929367243E'""" + +example_mode_aes_128_ecb_datatype_Float64_key_24_iv_None = r"""'3712B5B24D8F17CA7BE784AB7D57514E'""" + +example_mode_aes_128_ecb_datatype_Decimal32_key_24_iv_None = r"""'37BE2C0520C1A4C6F88CCBA2EEC020E5'""" + +example_mode_aes_128_ecb_datatype_Decimal64_key_24_iv_None = r"""'9ACC742A0FD36A80FEA2DBA2D73C4BFA'""" + +example_mode_aes_128_ecb_datatype_Decimal128_key_24_iv_None = r"""'EE8E9F15CAE8C5B6F2E0547636DEAF14644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_UUID_key_24_iv_None = r"""'6FD5580CC0B329922B575B79D4AD6E70644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_Date_key_24_iv_None = r"""'B0ADFA31B83C5B33B277097C33B06CB4'""" + +example_mode_aes_128_ecb_datatype_DateTime_key_24_iv_None = r"""'0632CF6416213B4153D247F4A85EAB19'""" + +example_mode_aes_128_ecb_datatype_DateTime64_key_24_iv_None = r"""'3D1049375E6EA3599E7AC2C753B1AA7F'""" + +example_mode_aes_128_ecb_datatype_LowCardinality_key_24_iv_None = r"""'697175286BC73A26C572DBD9480738F3'""" + +example_mode_aes_128_ecb_datatype_Array_key_24_iv_None = r"""'129253539B09D45C2147884BB2B866BD'""" + +example_mode_aes_128_ecb_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_128_ecb_datatype_IPv4_key_24_iv_None = r"""'B0D8302DB6F6B772E883A9353CDC28F0'""" + +example_mode_aes_128_ecb_datatype_IPv6_key_24_iv_None = r"""'781ABF0C605281F93A00B9BADD2CD1E3644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_Enum8_key_24_iv_None = r"""'113A22E44AC1C4B397CC8204C069F5F4'""" + +example_mode_aes_128_ecb_datatype_Enum16_key_24_iv_None = r"""'94DD57978311932F2E9FED922796A023'""" + +example_mode_aes_192_ecb_datatype_bytes_key_24_iv_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_ecb_datatype_emptystring_key_24_iv_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_utf8string_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_192_ecb_datatype_utf8fixedstring_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_192_ecb_datatype_String_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_FixedString_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_UInt8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_UInt16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_UInt32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_UInt64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Int8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Int16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_Int32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_Int64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Float32_key_24_iv_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_ecb_datatype_Float64_key_24_iv_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_ecb_datatype_Decimal32_key_24_iv_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_ecb_datatype_Decimal64_key_24_iv_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_ecb_datatype_Decimal128_key_24_iv_None = r"""'79207931793E374FB5A3A2AC1ECA857AD8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_UUID_key_24_iv_None = r"""'9FDB738E78D0D2F774C484ED82A854E4D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Date_key_24_iv_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_ecb_datatype_DateTime_key_24_iv_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_ecb_datatype_DateTime64_key_24_iv_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_ecb_datatype_LowCardinality_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_Array_key_24_iv_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_ecb_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_ecb_datatype_IPv4_key_24_iv_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_ecb_datatype_IPv6_key_24_iv_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F232D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Enum8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Enum16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_bytes_key_32_iv_None = r"""'4977677DCA6485E59B6D2AEC781DB50E'""" + +example_mode_aes_192_ecb_datatype_emptystring_key_32_iv_None = r"""'4F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_utf8string_key_32_iv_None = r"""'718D9D0FCE92DF4C73C6ADF970082475F5B91A80060E2F74BDB70F4D61D51128'""" + +example_mode_aes_192_ecb_datatype_utf8fixedstring_key_32_iv_None = r"""'718D9D0FCE92DF4C73C6ADF970082475F5B91A80060E2F74BDB70F4D61D51128'""" + +example_mode_aes_192_ecb_datatype_String_key_32_iv_None = r"""'851106E40808E28682DAC1AD840A7E92'""" + +example_mode_aes_192_ecb_datatype_FixedString_key_32_iv_None = r"""'851106E40808E28682DAC1AD840A7E92'""" + +example_mode_aes_192_ecb_datatype_UInt8_key_32_iv_None = r"""'077626FAA6FD46322732E0A107849CBE'""" + +example_mode_aes_192_ecb_datatype_UInt16_key_32_iv_None = r"""'C55FFF6925F48B2DDDD3F8696A6EE21A'""" + +example_mode_aes_192_ecb_datatype_UInt32_key_32_iv_None = r"""'7F3A3604968AC5BB1578D256A221442A'""" + +example_mode_aes_192_ecb_datatype_UInt64_key_32_iv_None = r"""'FCC1BC19F3A2F8F0484BD2BF3A069BB9'""" + +example_mode_aes_192_ecb_datatype_Int8_key_32_iv_None = r"""'077626FAA6FD46322732E0A107849CBE'""" + +example_mode_aes_192_ecb_datatype_Int16_key_32_iv_None = r"""'C55FFF6925F48B2DDDD3F8696A6EE21A'""" + +example_mode_aes_192_ecb_datatype_Int32_key_32_iv_None = r"""'7F3A3604968AC5BB1578D256A221442A'""" + +example_mode_aes_192_ecb_datatype_Int64_key_32_iv_None = r"""'FCC1BC19F3A2F8F0484BD2BF3A069BB9'""" + +example_mode_aes_192_ecb_datatype_Float32_key_32_iv_None = r"""'57AC570C827D0B10A340A635080E4BED'""" + +example_mode_aes_192_ecb_datatype_Float64_key_32_iv_None = r"""'EB761A3EB9A06676C875E70C6323B6D3'""" + +example_mode_aes_192_ecb_datatype_Decimal32_key_32_iv_None = r"""'07757D905A8DF38EEC7EAC2436BC883D'""" + +example_mode_aes_192_ecb_datatype_Decimal64_key_32_iv_None = r"""'B3DD7C625F5C9CAEB24B014AAF1660CF'""" + +example_mode_aes_192_ecb_datatype_Decimal128_key_32_iv_None = r"""'AF4EE89B714CBB6ED41802268A6C291F4F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_UUID_key_32_iv_None = r"""'6DDADFA4AD0FEA6DD4AE756F5E13E0EE4F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_Date_key_32_iv_None = r"""'D2212BB509C49D9DCBE970F86D34C4BE'""" + +example_mode_aes_192_ecb_datatype_DateTime_key_32_iv_None = r"""'C3AB19DCEFE1F61019484A2589D69037'""" + +example_mode_aes_192_ecb_datatype_DateTime64_key_32_iv_None = r"""'FB427597072B72E5C3D1D65247DB6A8C'""" + +example_mode_aes_192_ecb_datatype_LowCardinality_key_32_iv_None = r"""'851106E40808E28682DAC1AD840A7E92'""" + +example_mode_aes_192_ecb_datatype_Array_key_32_iv_None = r"""'D0FE4D4B34CFA03960FA609F1AA18D79'""" + +example_mode_aes_192_ecb_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_192_ecb_datatype_IPv4_key_32_iv_None = r"""'F07B737C70748F1ACDF5DBB874B4D78B'""" + +example_mode_aes_192_ecb_datatype_IPv6_key_32_iv_None = r"""'C95A36120328FDE78278655287FF91F44F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_Enum8_key_32_iv_None = r"""'077626FAA6FD46322732E0A107849CBE'""" + +example_mode_aes_192_ecb_datatype_Enum16_key_32_iv_None = r"""'C55FFF6925F48B2DDDD3F8696A6EE21A'""" + +example_mode_aes_256_ecb_datatype_bytes_key_32_iv_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_ecb_datatype_emptystring_key_32_iv_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_utf8string_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_256_ecb_datatype_utf8fixedstring_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_256_ecb_datatype_String_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_FixedString_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_UInt8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_UInt16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_UInt32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_UInt64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Int8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Int16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_Int32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_Int64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Float32_key_32_iv_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_ecb_datatype_Float64_key_32_iv_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_ecb_datatype_Decimal32_key_32_iv_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_ecb_datatype_Decimal64_key_32_iv_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_ecb_datatype_Decimal128_key_32_iv_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_UUID_key_32_iv_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C34217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Date_key_32_iv_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_ecb_datatype_DateTime_key_32_iv_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_ecb_datatype_DateTime64_key_32_iv_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_ecb_datatype_LowCardinality_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_Array_key_32_iv_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_ecb_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_ecb_datatype_IPv4_key_32_iv_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_ecb_datatype_IPv6_key_32_iv_None = r"""'8CBF2DC164F4086B8DD14B75E3065621217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Enum8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Enum16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_bytes_key_64_iv_None = r"""'2600697679EF0B3989C4EA3C0323CB8B'""" + +example_mode_aes_256_ecb_datatype_emptystring_key_64_iv_None = r"""'1F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_utf8string_key_64_iv_None = r"""'AB49D11AECD4A57A8BB4155C9F6733FBC9E3E51C40CDF6FA420050B461F48FA7'""" + +example_mode_aes_256_ecb_datatype_utf8fixedstring_key_64_iv_None = r"""'AB49D11AECD4A57A8BB4155C9F6733FBC9E3E51C40CDF6FA420050B461F48FA7'""" + +example_mode_aes_256_ecb_datatype_String_key_64_iv_None = r"""'7492B9A2D0E86DAF1DBCAEDBAD9E3D7E'""" + +example_mode_aes_256_ecb_datatype_FixedString_key_64_iv_None = r"""'7492B9A2D0E86DAF1DBCAEDBAD9E3D7E'""" + +example_mode_aes_256_ecb_datatype_UInt8_key_64_iv_None = r"""'FE2DE0EEF32A0510DC312ED77D1293EB'""" + +example_mode_aes_256_ecb_datatype_UInt16_key_64_iv_None = r"""'6805EDF8559E85ECBC4CA0AC3E241CB5'""" + +example_mode_aes_256_ecb_datatype_UInt32_key_64_iv_None = r"""'B2E6C9CE7EB187B7F56E754587C6BDBE'""" + +example_mode_aes_256_ecb_datatype_UInt64_key_64_iv_None = r"""'4F805F6D67E44124754951AEC9FDCEF3'""" + +example_mode_aes_256_ecb_datatype_Int8_key_64_iv_None = r"""'FE2DE0EEF32A0510DC312ED77D1293EB'""" + +example_mode_aes_256_ecb_datatype_Int16_key_64_iv_None = r"""'6805EDF8559E85ECBC4CA0AC3E241CB5'""" + +example_mode_aes_256_ecb_datatype_Int32_key_64_iv_None = r"""'B2E6C9CE7EB187B7F56E754587C6BDBE'""" + +example_mode_aes_256_ecb_datatype_Int64_key_64_iv_None = r"""'4F805F6D67E44124754951AEC9FDCEF3'""" + +example_mode_aes_256_ecb_datatype_Float32_key_64_iv_None = r"""'8014D3F1CFF0B7A66AE06FBDFA006FA6'""" + +example_mode_aes_256_ecb_datatype_Float64_key_64_iv_None = r"""'2C734E3A5A82C65E8918FA329B936114'""" + +example_mode_aes_256_ecb_datatype_Decimal32_key_64_iv_None = r"""'95245BC3292B4749E8CC7B5FDD26CAD9'""" + +example_mode_aes_256_ecb_datatype_Decimal64_key_64_iv_None = r"""'443682D0541F666078718D4790C3CE4E'""" + +example_mode_aes_256_ecb_datatype_Decimal128_key_64_iv_None = r"""'BE7F01084B171062CB4CCCCF9BB77D671F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_UUID_key_64_iv_None = r"""'65F2B1003C30A2E7148652BA06EF09FC1F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_Date_key_64_iv_None = r"""'D72724C85F90712153FC49FB33432644'""" + +example_mode_aes_256_ecb_datatype_DateTime_key_64_iv_None = r"""'B734143D37365E5C3325E0396BABC2AB'""" + +example_mode_aes_256_ecb_datatype_DateTime64_key_64_iv_None = r"""'78DDC273BE606E8F546538FC02508360'""" + +example_mode_aes_256_ecb_datatype_LowCardinality_key_64_iv_None = r"""'7492B9A2D0E86DAF1DBCAEDBAD9E3D7E'""" + +example_mode_aes_256_ecb_datatype_Array_key_64_iv_None = r"""'592C364BE5AEBE911096DEB1F6C75AB9'""" + +example_mode_aes_256_ecb_datatype_NULL_key_64_iv_None = r"""'\\N'""" + +example_mode_aes_256_ecb_datatype_IPv4_key_64_iv_None = r"""'F5401B3B979784C3FF4C86DD726872F9'""" + +example_mode_aes_256_ecb_datatype_IPv6_key_64_iv_None = r"""'3F810185F7D07B5E5A897E96BCA930ED1F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_Enum8_key_64_iv_None = r"""'FE2DE0EEF32A0510DC312ED77D1293EB'""" + +example_mode_aes_256_ecb_datatype_Enum16_key_64_iv_None = r"""'6805EDF8559E85ECBC4CA0AC3E241CB5'""" + +example_mode_aes_128_cbc_datatype_bytes_key_16_iv_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_cbc_datatype_emptystring_key_16_iv_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_cbc_datatype_utf8string_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_128_cbc_datatype_String_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_FixedString_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_UInt8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_UInt16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_UInt32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_UInt64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Int8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Int16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_Int32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_Int64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Float32_key_16_iv_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_cbc_datatype_Float64_key_16_iv_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_cbc_datatype_Decimal32_key_16_iv_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_cbc_datatype_Decimal64_key_16_iv_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_cbc_datatype_Decimal128_key_16_iv_None = r"""'5F3DBFA74809E45E03980357B26787AF0D55B905F5525D3F5916FF811D8A6E7E'""" + +example_mode_aes_128_cbc_datatype_UUID_key_16_iv_None = r"""'FF9161B222B4A67481271035745F06D991B6833DF67CBA9BC6E1AAEADBE363BB'""" + +example_mode_aes_128_cbc_datatype_Date_key_16_iv_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_cbc_datatype_DateTime_key_16_iv_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_cbc_datatype_DateTime64_key_16_iv_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_Array_key_16_iv_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_cbc_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_key_16_iv_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_cbc_datatype_IPv6_key_16_iv_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9B1B4BE8B15BAE0B2C9196D69E3D53C6C'""" + +example_mode_aes_128_cbc_datatype_Enum8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Enum16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_192_cbc_datatype_bytes_key_24_iv_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_cbc_datatype_emptystring_key_24_iv_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_cbc_datatype_utf8string_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_192_cbc_datatype_String_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_FixedString_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_UInt8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_UInt16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_UInt32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_UInt64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Int8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Int16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_Int32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_Int64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Float32_key_24_iv_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_cbc_datatype_Float64_key_24_iv_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_cbc_datatype_Decimal32_key_24_iv_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_cbc_datatype_Decimal64_key_24_iv_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_cbc_datatype_Decimal128_key_24_iv_None = r"""'79207931793E374FB5A3A2AC1ECA857A583603B3047000A843425EECA4C35311'""" + +example_mode_aes_192_cbc_datatype_UUID_key_24_iv_None = r"""'9FDB738E78D0D2F774C484ED82A854E46B580C61DBE08478DC523DA6AD605078'""" + +example_mode_aes_192_cbc_datatype_Date_key_24_iv_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_cbc_datatype_DateTime_key_24_iv_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_cbc_datatype_DateTime64_key_24_iv_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_Array_key_24_iv_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_cbc_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_key_24_iv_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_cbc_datatype_IPv6_key_24_iv_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F2326879326F3973401A6293A92BCB8EDFC4'""" + +example_mode_aes_192_cbc_datatype_Enum8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Enum16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_256_cbc_datatype_bytes_key_32_iv_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_cbc_datatype_emptystring_key_32_iv_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_cbc_datatype_utf8string_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_256_cbc_datatype_String_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_FixedString_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_UInt8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_UInt16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_UInt32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_UInt64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Int8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Int16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_Int32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_Int64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Float32_key_32_iv_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_cbc_datatype_Float64_key_32_iv_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_cbc_datatype_Decimal32_key_32_iv_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_cbc_datatype_Decimal64_key_32_iv_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_cbc_datatype_Decimal128_key_32_iv_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB36585C228B0286E7A8D8DBAF754C4C38'""" + +example_mode_aes_256_cbc_datatype_UUID_key_32_iv_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C3428548E6132D69A22B320775A21ABA11F'""" + +example_mode_aes_256_cbc_datatype_Date_key_32_iv_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_cbc_datatype_DateTime_key_32_iv_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_cbc_datatype_DateTime64_key_32_iv_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_Array_key_32_iv_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_cbc_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_key_32_iv_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_cbc_datatype_IPv6_key_32_iv_None = r"""'8CBF2DC164F4086B8DD14B75E3065621393DE8421BAA5AE5E87096AEA7087507'""" + +example_mode_aes_256_cbc_datatype_Enum8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Enum16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_128_cbc_datatype_bytes_key_16_iv_16 = r"""'CDA4B7027137998B9A33C2096C9A50DD'""" + +example_mode_aes_128_cbc_datatype_emptystring_key_16_iv_16 = r"""'56A77308430BA344FFBF016999795ED5'""" + +example_mode_aes_128_cbc_datatype_utf8string_key_16_iv_16 = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_key_16_iv_16 = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_128_cbc_datatype_String_key_16_iv_16 = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_FixedString_key_16_iv_16 = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_UInt8_key_16_iv_16 = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_UInt16_key_16_iv_16 = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_UInt32_key_16_iv_16 = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_UInt64_key_16_iv_16 = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Int8_key_16_iv_16 = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Int16_key_16_iv_16 = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_Int32_key_16_iv_16 = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_Int64_key_16_iv_16 = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Float32_key_16_iv_16 = r"""'A5425BDEB6B83E311C45249DAF3153F5'""" + +example_mode_aes_128_cbc_datatype_Float64_key_16_iv_16 = r"""'EEDA98EC4045C7D351F3905313073B79'""" + +example_mode_aes_128_cbc_datatype_Decimal32_key_16_iv_16 = r"""'52EBB74292ECD37A29E9809166CC77DB'""" + +example_mode_aes_128_cbc_datatype_Decimal64_key_16_iv_16 = r"""'95EF455767EC8FBD32BAAEFFB44FEEB7'""" + +example_mode_aes_128_cbc_datatype_Decimal128_key_16_iv_16 = r"""'94C066884FA09B0D3C750F20A2823304A2FE20B6B69AB18373E3F58623E0D7FB'""" + +example_mode_aes_128_cbc_datatype_UUID_key_16_iv_16 = r"""'1D909C15BB882E89AD68B1EFEAC72148DCD05E2303B6BE19007A945AFB778B42'""" + +example_mode_aes_128_cbc_datatype_Date_key_16_iv_16 = r"""'24A4F8CE8A9FAE48A0AFEB8A6203EFEA'""" + +example_mode_aes_128_cbc_datatype_DateTime_key_16_iv_16 = r"""'0DD5554819E3995B1B6B00362AEE9424'""" + +example_mode_aes_128_cbc_datatype_DateTime64_key_16_iv_16 = r"""'0E55319903957C9D1FDA4FB65C3871CB'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_key_16_iv_16 = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_Array_key_16_iv_16 = r"""'D53C82A5D13256B88DF41C1C1D924E40'""" + +example_mode_aes_128_cbc_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_key_16_iv_16 = r"""'C0D81AAB3134EAB5B1F190958C6A29F9'""" + +example_mode_aes_128_cbc_datatype_IPv6_key_16_iv_16 = r"""'AE1A36F75C9BB387121445069A9968CA247FA4459ED3C8809089FEE334EB1EC7'""" + +example_mode_aes_128_cbc_datatype_Enum8_key_16_iv_16 = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Enum16_key_16_iv_16 = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_bytes_key_24_iv_24 = r"""'FD4D81969EDCB22A5B5DE4E21BDFE267'""" + +example_mode_aes_128_cbc_datatype_emptystring_key_24_iv_24 = r"""'BEFE724909AC17B32920D0400312227E'""" + +example_mode_aes_128_cbc_datatype_utf8string_key_24_iv_24 = r"""'A4FBBF549A9D453212CB69882DF12D34DAA659D7D176B78DE14F6AF36E0BE2C9'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_key_24_iv_24 = r"""'A4FBBF549A9D453212CB69882DF12D34DAA659D7D176B78DE14F6AF36E0BE2C9'""" + +example_mode_aes_128_cbc_datatype_String_key_24_iv_24 = r"""'73C9874744984892250CCCEC8541D690'""" + +example_mode_aes_128_cbc_datatype_FixedString_key_24_iv_24 = r"""'73C9874744984892250CCCEC8541D690'""" + +example_mode_aes_128_cbc_datatype_UInt8_key_24_iv_24 = r"""'0E97B8F125240D96125B8AC23A798981'""" + +example_mode_aes_128_cbc_datatype_UInt16_key_24_iv_24 = r"""'9118F2C6B4E17730C62F85CB22E9A446'""" + +example_mode_aes_128_cbc_datatype_UInt32_key_24_iv_24 = r"""'2A189EB114A9487142573673C0F30929'""" + +example_mode_aes_128_cbc_datatype_UInt64_key_24_iv_24 = r"""'B6CA3E3C1C7830203FABD429664E81B3'""" + +example_mode_aes_128_cbc_datatype_Int8_key_24_iv_24 = r"""'0E97B8F125240D96125B8AC23A798981'""" + +example_mode_aes_128_cbc_datatype_Int16_key_24_iv_24 = r"""'9118F2C6B4E17730C62F85CB22E9A446'""" + +example_mode_aes_128_cbc_datatype_Int32_key_24_iv_24 = r"""'2A189EB114A9487142573673C0F30929'""" + +example_mode_aes_128_cbc_datatype_Int64_key_24_iv_24 = r"""'B6CA3E3C1C7830203FABD429664E81B3'""" + +example_mode_aes_128_cbc_datatype_Float32_key_24_iv_24 = r"""'79B44B6E26BC4BBACBBE312329F8C86D'""" + +example_mode_aes_128_cbc_datatype_Float64_key_24_iv_24 = r"""'C23864E616278AD006ED806C1EF89F71'""" + +example_mode_aes_128_cbc_datatype_Decimal32_key_24_iv_24 = r"""'4E14763E17CD954A2754768FF664CECE'""" + +example_mode_aes_128_cbc_datatype_Decimal64_key_24_iv_24 = r"""'A5D655F79EA638ECD6879A515853DD50'""" + +example_mode_aes_128_cbc_datatype_Decimal128_key_24_iv_24 = r"""'5C1C88BE7B68B91167BD317D3E6291A0D7920AEDA123237D0223EFB15F9C8ADC'""" + +example_mode_aes_128_cbc_datatype_UUID_key_24_iv_24 = r"""'ADF1CF6C8AF83CBD4BB3313E7E88FE8C377730890CC89DD91E95436ABC5E4F3A'""" + +example_mode_aes_128_cbc_datatype_Date_key_24_iv_24 = r"""'3127F80211A13ED02CED473BF2BFC28B'""" + +example_mode_aes_128_cbc_datatype_DateTime_key_24_iv_24 = r"""'A6B567CCFCE4F693BD6575D2D4DF498B'""" + +example_mode_aes_128_cbc_datatype_DateTime64_key_24_iv_24 = r"""'410F1A2A9E39722A1D0B549C3ADF9526'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_key_24_iv_24 = r"""'73C9874744984892250CCCEC8541D690'""" + +example_mode_aes_128_cbc_datatype_Array_key_24_iv_24 = r"""'14A8202D2CACBE584BE9313C72F0478D'""" + +example_mode_aes_128_cbc_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_key_24_iv_24 = r"""'9C554B2C04C5C9EAC0E7E36F1BD5CB80'""" + +example_mode_aes_128_cbc_datatype_IPv6_key_24_iv_24 = r"""'D548D293E5F41FE72A940A6A29EDBCDA8AAEBAC8BA27FB0B4DB65DAD63895B9F'""" + +example_mode_aes_128_cbc_datatype_Enum8_key_24_iv_24 = r"""'0E97B8F125240D96125B8AC23A798981'""" + +example_mode_aes_128_cbc_datatype_Enum16_key_24_iv_24 = r"""'9118F2C6B4E17730C62F85CB22E9A446'""" + +example_mode_aes_192_cbc_datatype_bytes_key_24_iv_16 = r"""'67771349942D4F812553F2E1E3FFB276'""" + +example_mode_aes_192_cbc_datatype_emptystring_key_24_iv_16 = r"""'62E9214DB5E239F0CAD31ADF26AB313F'""" + +example_mode_aes_192_cbc_datatype_utf8string_key_24_iv_16 = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_key_24_iv_16 = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_192_cbc_datatype_String_key_24_iv_16 = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_FixedString_key_24_iv_16 = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_UInt8_key_24_iv_16 = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_UInt16_key_24_iv_16 = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_UInt32_key_24_iv_16 = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_UInt64_key_24_iv_16 = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Int8_key_24_iv_16 = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Int16_key_24_iv_16 = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_Int32_key_24_iv_16 = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_Int64_key_24_iv_16 = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Float32_key_24_iv_16 = r"""'62EBE4FD1035D405BBD6C41436780E13'""" + +example_mode_aes_192_cbc_datatype_Float64_key_24_iv_16 = r"""'5706FC9892A4C1AB48FC93E13C9C72FE'""" + +example_mode_aes_192_cbc_datatype_Decimal32_key_24_iv_16 = r"""'BB056843D369A5E55982C92AD52EEC07'""" + +example_mode_aes_192_cbc_datatype_Decimal64_key_24_iv_16 = r"""'70ACD4156F9AC1444A75EFCB9202CA00'""" + +example_mode_aes_192_cbc_datatype_Decimal128_key_24_iv_16 = r"""'04748A45840A0CAAC83F139DB01C504B01FC56631A8B2FFBE68F2FC85B6FEEDE'""" + +example_mode_aes_192_cbc_datatype_UUID_key_24_iv_16 = r"""'D7B2ABC08F67823F61C3E8F680C12B3A8AA3E3711D412CB55ACFBC89C14949A8'""" + +example_mode_aes_192_cbc_datatype_Date_key_24_iv_16 = r"""'734BBE526E56B280E90E53DDEA7DB69B'""" + +example_mode_aes_192_cbc_datatype_DateTime_key_24_iv_16 = r"""'9B9BE7CC20F75DA3F39F688DE3A1ADAA'""" + +example_mode_aes_192_cbc_datatype_DateTime64_key_24_iv_16 = r"""'554FCAAF985378A561F7C6ED91E20C89'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_key_24_iv_16 = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_Array_key_24_iv_16 = r"""'D85AF1078F110329896EFC462340171E'""" + +example_mode_aes_192_cbc_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_key_24_iv_16 = r"""'6AF45078B1E924B6C107D4C0236EA937'""" + +example_mode_aes_192_cbc_datatype_IPv6_key_24_iv_16 = r"""'9E4F8E54B265A340090DC7FE4F53BB50048442F5632A7B1630AE80DFD938E9AA'""" + +example_mode_aes_192_cbc_datatype_Enum8_key_24_iv_16 = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Enum16_key_24_iv_16 = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_bytes_key_32_iv_32 = r"""'34FFC84A0D7CAD43F6BDB7F0C57CE511'""" + +example_mode_aes_192_cbc_datatype_emptystring_key_32_iv_32 = r"""'B9937ECA1C6EADCCA37CD7E8BA89D939'""" + +example_mode_aes_192_cbc_datatype_utf8string_key_32_iv_32 = r"""'00A1C691A68F5BF50BA973D72A5DBFBD579279FD953AE222F41FD0DAF48A3C90'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_key_32_iv_32 = r"""'00A1C691A68F5BF50BA973D72A5DBFBD579279FD953AE222F41FD0DAF48A3C90'""" + +example_mode_aes_192_cbc_datatype_String_key_32_iv_32 = r"""'508551DA505F6538F90DC607423CFAD4'""" + +example_mode_aes_192_cbc_datatype_FixedString_key_32_iv_32 = r"""'508551DA505F6538F90DC607423CFAD4'""" + +example_mode_aes_192_cbc_datatype_UInt8_key_32_iv_32 = r"""'C2D3A1603DBAC305D8180F85A3830300'""" + +example_mode_aes_192_cbc_datatype_UInt16_key_32_iv_32 = r"""'A7EEA5561C4BB75015632824CAE9AC1E'""" + +example_mode_aes_192_cbc_datatype_UInt32_key_32_iv_32 = r"""'97EE21986E9A9F3A4F67851D05C93830'""" + +example_mode_aes_192_cbc_datatype_UInt64_key_32_iv_32 = r"""'474F106EAC813E4B432B2CA58D11F0E1'""" + +example_mode_aes_192_cbc_datatype_Int8_key_32_iv_32 = r"""'C2D3A1603DBAC305D8180F85A3830300'""" + +example_mode_aes_192_cbc_datatype_Int16_key_32_iv_32 = r"""'A7EEA5561C4BB75015632824CAE9AC1E'""" + +example_mode_aes_192_cbc_datatype_Int32_key_32_iv_32 = r"""'97EE21986E9A9F3A4F67851D05C93830'""" + +example_mode_aes_192_cbc_datatype_Int64_key_32_iv_32 = r"""'474F106EAC813E4B432B2CA58D11F0E1'""" + +example_mode_aes_192_cbc_datatype_Float32_key_32_iv_32 = r"""'3C76B134B1FF458CBE4430DCDF3EE8F9'""" + +example_mode_aes_192_cbc_datatype_Float64_key_32_iv_32 = r"""'0C2C54B82F3B7A889696EACBBBC5AA96'""" + +example_mode_aes_192_cbc_datatype_Decimal32_key_32_iv_32 = r"""'A038F21DD2F3C6FE3382C849E6BCC6B8'""" + +example_mode_aes_192_cbc_datatype_Decimal64_key_32_iv_32 = r"""'CE5E44EF4BAC3455C1FE57C82498FC32'""" + +example_mode_aes_192_cbc_datatype_Decimal128_key_32_iv_32 = r"""'DF51ED3C3BE2A229F07C4C9BB64902DD9383D2999BDD4E3FEFE023671B90CA88'""" + +example_mode_aes_192_cbc_datatype_UUID_key_32_iv_32 = r"""'B4A04107B9945309227C08EEE5516367F2C64F1587FEB4720620AA04485CA18C'""" + +example_mode_aes_192_cbc_datatype_Date_key_32_iv_32 = r"""'60253CED16ECAB1D34A5CC131E55B769'""" + +example_mode_aes_192_cbc_datatype_DateTime_key_32_iv_32 = r"""'99C6EE6DF7A06D02FE4D60018784F129'""" + +example_mode_aes_192_cbc_datatype_DateTime64_key_32_iv_32 = r"""'C0329F9CB5CFDDDEA74275618A099FAE'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_key_32_iv_32 = r"""'508551DA505F6538F90DC607423CFAD4'""" + +example_mode_aes_192_cbc_datatype_Array_key_32_iv_32 = r"""'53161975B2A33AAFB8379B39B255D45A'""" + +example_mode_aes_192_cbc_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_key_32_iv_32 = r"""'2B2ACE9C2914F3F2F9E206D25CD29429'""" + +example_mode_aes_192_cbc_datatype_IPv6_key_32_iv_32 = r"""'16AF2466967A2705EC5D588C0C155457A3EC9F82C1C5FB98CE86AC5697EF7223'""" + +example_mode_aes_192_cbc_datatype_Enum8_key_32_iv_32 = r"""'C2D3A1603DBAC305D8180F85A3830300'""" + +example_mode_aes_192_cbc_datatype_Enum16_key_32_iv_32 = r"""'A7EEA5561C4BB75015632824CAE9AC1E'""" + +example_mode_aes_256_cbc_datatype_bytes_key_32_iv_16 = r"""'6046ECF8094941C6DEC9278FF6F137E9'""" + +example_mode_aes_256_cbc_datatype_emptystring_key_32_iv_16 = r"""'4EC7785DA650D55B71B52816B1DB5AD3'""" + +example_mode_aes_256_cbc_datatype_utf8string_key_32_iv_16 = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_key_32_iv_16 = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_256_cbc_datatype_String_key_32_iv_16 = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_FixedString_key_32_iv_16 = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_UInt8_key_32_iv_16 = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_UInt16_key_32_iv_16 = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_UInt32_key_32_iv_16 = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_UInt64_key_32_iv_16 = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Int8_key_32_iv_16 = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Int16_key_32_iv_16 = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_Int32_key_32_iv_16 = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_Int64_key_32_iv_16 = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Float32_key_32_iv_16 = r"""'E639AA3E45D8C2759181FD736CD58EDC'""" + +example_mode_aes_256_cbc_datatype_Float64_key_32_iv_16 = r"""'CFEF3FDC054997559DF5DCFB5F215B58'""" + +example_mode_aes_256_cbc_datatype_Decimal32_key_32_iv_16 = r"""'E2F57A092A1759D39F4AE67C9543FAB8'""" + +example_mode_aes_256_cbc_datatype_Decimal64_key_32_iv_16 = r"""'6259A2CFD3D83352A44C03DB050077B3'""" + +example_mode_aes_256_cbc_datatype_Decimal128_key_32_iv_16 = r"""'AEC71CA2D87098392689F9EB2ED93A84FA5787E643E28CB3C2013F8FCC24E387'""" + +example_mode_aes_256_cbc_datatype_UUID_key_32_iv_16 = r"""'88BA86B14A468DC92084B7152B172E142D88CBFB639A8FF2F480F1727972251C'""" + +example_mode_aes_256_cbc_datatype_Date_key_32_iv_16 = r"""'C67C84B1C6BF4527A7E730499FF39C86'""" + +example_mode_aes_256_cbc_datatype_DateTime_key_32_iv_16 = r"""'7FDC1B0797A5F3C04CDA82729A1EA4AA'""" + +example_mode_aes_256_cbc_datatype_DateTime64_key_32_iv_16 = r"""'B1B7401FB2B65BCB3448C1BE179F6AA6'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_key_32_iv_16 = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_Array_key_32_iv_16 = r"""'6BB1E8429CC612B0AA74282B81D4FE8A'""" + +example_mode_aes_256_cbc_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_key_32_iv_16 = r"""'51364C8DC6882CA1F03CF7FB45117EEF'""" + +example_mode_aes_256_cbc_datatype_IPv6_key_32_iv_16 = r"""'87A1C4D4672EFE64DC98E040EAD6B3126C899C263577B3D8EE8A3952BE5CDC1B'""" + +example_mode_aes_256_cbc_datatype_Enum8_key_32_iv_16 = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Enum16_key_32_iv_16 = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_bytes_key_64_iv_64 = r"""'BC91CDADF0BD3A8E9FF014B1E494FE7E'""" + +example_mode_aes_256_cbc_datatype_emptystring_key_64_iv_64 = r"""'A574B24357860C4B9C580D4CBE6C22C7'""" + +example_mode_aes_256_cbc_datatype_utf8string_key_64_iv_64 = r"""'3DA0F0C80E8067C1A0B5C82EF7A2D12A1EAD3C258827367A46D13D522B4D85DC'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_key_64_iv_64 = r"""'3DA0F0C80E8067C1A0B5C82EF7A2D12A1EAD3C258827367A46D13D522B4D85DC'""" + +example_mode_aes_256_cbc_datatype_String_key_64_iv_64 = r"""'4CE9C9AFDC1E1E1EF2D1F4C141CE1874'""" + +example_mode_aes_256_cbc_datatype_FixedString_key_64_iv_64 = r"""'4CE9C9AFDC1E1E1EF2D1F4C141CE1874'""" + +example_mode_aes_256_cbc_datatype_UInt8_key_64_iv_64 = r"""'89E81101BF6A6E1DBEB6C0EAA67BEF14'""" + +example_mode_aes_256_cbc_datatype_UInt16_key_64_iv_64 = r"""'3124E8A62599D6C8A0C16A7ED0F01C58'""" + +example_mode_aes_256_cbc_datatype_UInt32_key_64_iv_64 = r"""'DD8866C2B2E443073D2F7B5A6FEE92E9'""" + +example_mode_aes_256_cbc_datatype_UInt64_key_64_iv_64 = r"""'C45A6D56D06F8A3A6A753D9BADBE88A4'""" + +example_mode_aes_256_cbc_datatype_Int8_key_64_iv_64 = r"""'89E81101BF6A6E1DBEB6C0EAA67BEF14'""" + +example_mode_aes_256_cbc_datatype_Int16_key_64_iv_64 = r"""'3124E8A62599D6C8A0C16A7ED0F01C58'""" + +example_mode_aes_256_cbc_datatype_Int32_key_64_iv_64 = r"""'DD8866C2B2E443073D2F7B5A6FEE92E9'""" + +example_mode_aes_256_cbc_datatype_Int64_key_64_iv_64 = r"""'C45A6D56D06F8A3A6A753D9BADBE88A4'""" + +example_mode_aes_256_cbc_datatype_Float32_key_64_iv_64 = r"""'66778B8F543C2C8402865CFB41187A09'""" + +example_mode_aes_256_cbc_datatype_Float64_key_64_iv_64 = r"""'F05D98EC189434B1F9177D0915FE939F'""" + +example_mode_aes_256_cbc_datatype_Decimal32_key_64_iv_64 = r"""'81190FEC7ADDBD8B919DD85AD71FF0B3'""" + +example_mode_aes_256_cbc_datatype_Decimal64_key_64_iv_64 = r"""'AB8B69F837890ED5684B071FF2359BED'""" + +example_mode_aes_256_cbc_datatype_Decimal128_key_64_iv_64 = r"""'765BDA3E9FA6177EB1839155F9C55A856315AB458B1DC334DF821A6FD768BFC8'""" + +example_mode_aes_256_cbc_datatype_UUID_key_64_iv_64 = r"""'99588BF38D9D6A2ABBEBD22E02F4D0F1CD5787AF78FE18679B8CF02A53D2600E'""" + +example_mode_aes_256_cbc_datatype_Date_key_64_iv_64 = r"""'ABEEA4FD2E771A3936EB591F674E4CDA'""" + +example_mode_aes_256_cbc_datatype_DateTime_key_64_iv_64 = r"""'C7EC851BCD2FBA03E58AC9F027E06444'""" + +example_mode_aes_256_cbc_datatype_DateTime64_key_64_iv_64 = r"""'53D42045D02E68F0D565EB8E093C83B8'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_key_64_iv_64 = r"""'4CE9C9AFDC1E1E1EF2D1F4C141CE1874'""" + +example_mode_aes_256_cbc_datatype_Array_key_64_iv_64 = r"""'5A3063DCD98C7C59B41068F26DA0CC52'""" + +example_mode_aes_256_cbc_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_key_64_iv_64 = r"""'1F12AE92E01133B2BB3F4F5AC6041FC9'""" + +example_mode_aes_256_cbc_datatype_IPv6_key_64_iv_64 = r"""'32406EB5546AECA4EE235A586577B14A5F6070FE879D6E07E6431748E3C2E80F'""" + +example_mode_aes_256_cbc_datatype_Enum8_key_64_iv_64 = r"""'89E81101BF6A6E1DBEB6C0EAA67BEF14'""" + +example_mode_aes_256_cbc_datatype_Enum16_key_64_iv_64 = r"""'3124E8A62599D6C8A0C16A7ED0F01C58'""" + +example_mode_aes_128_cfb1_datatype_bytes_key_16_iv_None = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_cfb1_datatype_utf8string_key_16_iv_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_key_16_iv_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_128_cfb1_datatype_String_key_16_iv_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_FixedString_key_16_iv_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_UInt8_key_16_iv_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_key_16_iv_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_UInt32_key_16_iv_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_UInt64_key_16_iv_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Int8_key_16_iv_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_key_16_iv_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_Int32_key_16_iv_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_Int64_key_16_iv_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Float32_key_16_iv_None = r"""'0000B9AB'""" + +example_mode_aes_128_cfb1_datatype_Float64_key_16_iv_None = r"""'000000000000FFF6'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_key_16_iv_None = r"""'2E09CA6A'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_key_16_iv_None = r"""'2E09CA6A6DBEE799'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_key_16_iv_None = r"""'2E09CA6A6DBEE79923BA65C6B78FD199'""" + +example_mode_aes_128_cfb1_datatype_UUID_key_16_iv_None = r"""'E590DFB515D3A518F85C66A6A5EC9C6E'""" + +example_mode_aes_128_cfb1_datatype_Date_key_16_iv_None = r"""'42F0'""" + +example_mode_aes_128_cfb1_datatype_DateTime_key_16_iv_None = r"""'5475EC3D'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_key_16_iv_None = r"""'21CDF1128AE44A37'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_key_16_iv_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_Array_key_16_iv_None = r"""'0170'""" + +example_mode_aes_128_cfb1_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_key_16_iv_None = r"""'240A9E43'""" + +example_mode_aes_128_cfb1_datatype_IPv6_key_16_iv_None = r"""'2E642EF4B07D9B1251BE3B3CBDBCC6F6'""" + +example_mode_aes_128_cfb1_datatype_Enum8_key_16_iv_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_key_16_iv_None = r"""'0173'""" + +example_mode_aes_192_cfb1_datatype_bytes_key_24_iv_None = r"""'00'""" + +example_mode_aes_192_cfb1_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_cfb1_datatype_utf8string_key_24_iv_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_key_24_iv_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_192_cfb1_datatype_String_key_24_iv_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_FixedString_key_24_iv_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_UInt8_key_24_iv_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_UInt16_key_24_iv_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_UInt32_key_24_iv_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_UInt64_key_24_iv_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Int8_key_24_iv_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Int16_key_24_iv_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_Int32_key_24_iv_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_Int64_key_24_iv_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Float32_key_24_iv_None = r"""'0000FCAE'""" + +example_mode_aes_192_cfb1_datatype_Float64_key_24_iv_None = r"""'000000000000A79C'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_key_24_iv_None = r"""'3F406C3F'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_key_24_iv_None = r"""'3F406C3F3A41B134'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_key_24_iv_None = r"""'3F406C3F3A41B134310D6B68BEBC5708'""" + +example_mode_aes_192_cfb1_datatype_UUID_key_24_iv_None = r"""'B7F80F1BDCA1C4193E5AB11078FEA213'""" + +example_mode_aes_192_cfb1_datatype_Date_key_24_iv_None = r"""'6FF6'""" + +example_mode_aes_192_cfb1_datatype_DateTime_key_24_iv_None = r"""'7013E555'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_key_24_iv_None = r"""'371AF0291536F5B7'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_key_24_iv_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_Array_key_24_iv_None = r"""'01FA'""" + +example_mode_aes_192_cfb1_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_key_24_iv_None = r"""'33895F70'""" + +example_mode_aes_192_cfb1_datatype_IPv6_key_24_iv_None = r"""'3F24552946522B931290F904186B055A'""" + +example_mode_aes_192_cfb1_datatype_Enum8_key_24_iv_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Enum16_key_24_iv_None = r"""'01F9'""" + +example_mode_aes_256_cfb1_datatype_bytes_key_32_iv_None = r"""'B8'""" + +example_mode_aes_256_cfb1_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_cfb1_datatype_utf8string_key_32_iv_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_key_32_iv_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_256_cfb1_datatype_String_key_32_iv_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_FixedString_key_32_iv_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_UInt8_key_32_iv_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_UInt16_key_32_iv_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_UInt32_key_32_iv_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_UInt64_key_32_iv_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Int8_key_32_iv_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Int16_key_32_iv_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_Int32_key_32_iv_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_Int64_key_32_iv_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Float32_key_32_iv_None = r"""'B85F0E63'""" + +example_mode_aes_256_cfb1_datatype_Float64_key_32_iv_None = r"""'B85FDB5A8FE0C0BB'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_key_32_iv_None = r"""'891B85B3'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_key_32_iv_None = r"""'891B85B3C1BA6EE1'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_key_32_iv_None = r"""'891B85B3C1BA6EE137EF658F618D1F3F'""" + +example_mode_aes_256_cfb1_datatype_UUID_key_32_iv_None = r"""'121B5EE9929417BC1CDBDB390BC93B4A'""" + +example_mode_aes_256_cfb1_datatype_Date_key_32_iv_None = r"""'D40F'""" + +example_mode_aes_256_cfb1_datatype_DateTime_key_32_iv_None = r"""'CF27297C'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_key_32_iv_None = r"""'8773F350CD394D36'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_key_32_iv_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_Array_key_32_iv_None = r"""'B9EE'""" + +example_mode_aes_256_cfb1_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_key_32_iv_None = r"""'8383FD3C'""" + +example_mode_aes_256_cfb1_datatype_IPv6_key_32_iv_None = r"""'897A84A02FD451D3DDB92FF290BF9B7C'""" + +example_mode_aes_256_cfb1_datatype_Enum8_key_32_iv_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Enum16_key_32_iv_None = r"""'B9ED'""" + +example_mode_aes_128_cfb1_datatype_bytes_key_16_iv_16 = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_cfb1_datatype_utf8string_key_16_iv_16 = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_key_16_iv_16 = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_128_cfb1_datatype_String_key_16_iv_16 = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_FixedString_key_16_iv_16 = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_UInt8_key_16_iv_16 = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_key_16_iv_16 = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_UInt32_key_16_iv_16 = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_UInt64_key_16_iv_16 = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Int8_key_16_iv_16 = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_key_16_iv_16 = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_Int32_key_16_iv_16 = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_Int64_key_16_iv_16 = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Float32_key_16_iv_16 = r"""'00B931F2'""" + +example_mode_aes_128_cfb1_datatype_Float64_key_16_iv_16 = r"""'00B99AAE199C3C93'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_key_16_iv_16 = r"""'2D557511'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_key_16_iv_16 = r"""'2D557511511F90FB'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_key_16_iv_16 = r"""'2D557511511F90FBC464352E8A02FC51'""" + +example_mode_aes_128_cfb1_datatype_UUID_key_16_iv_16 = r"""'8AE269086C72AD682EB92ABA6CA58E49'""" + +example_mode_aes_128_cfb1_datatype_Date_key_16_iv_16 = r"""'5FC9'""" + +example_mode_aes_128_cfb1_datatype_DateTime_key_16_iv_16 = r"""'42970865'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_key_16_iv_16 = r"""'20B310A2F7EF8460'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_key_16_iv_16 = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_Array_key_16_iv_16 = r"""'018A'""" + +example_mode_aes_128_cfb1_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_key_16_iv_16 = r"""'27476DAF'""" + +example_mode_aes_128_cfb1_datatype_IPv6_key_16_iv_16 = r"""'2D311FBDC0A5C652AAD863398F94C5C3'""" + +example_mode_aes_128_cfb1_datatype_Enum8_key_16_iv_16 = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_key_16_iv_16 = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_bytes_key_24_iv_24 = r"""'1C'""" + +example_mode_aes_128_cfb1_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_cfb1_datatype_utf8string_key_24_iv_24 = r"""'612A502D86BAEA94E98C1381AE057BD8606FB3776DC79E52'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_key_24_iv_24 = r"""'612A502D86BAEA94E98C1381AE057BD8606FB3776DC79E52'""" + +example_mode_aes_128_cfb1_datatype_String_key_24_iv_24 = r"""'30'""" + +example_mode_aes_128_cfb1_datatype_FixedString_key_24_iv_24 = r"""'30'""" + +example_mode_aes_128_cfb1_datatype_UInt8_key_24_iv_24 = r"""'1D'""" + +example_mode_aes_128_cfb1_datatype_UInt16_key_24_iv_24 = r"""'1D52'""" + +example_mode_aes_128_cfb1_datatype_UInt32_key_24_iv_24 = r"""'1D52BAD5'""" + +example_mode_aes_128_cfb1_datatype_UInt64_key_24_iv_24 = r"""'1D52BAD5E209AC18'""" + +example_mode_aes_128_cfb1_datatype_Int8_key_24_iv_24 = r"""'1D'""" + +example_mode_aes_128_cfb1_datatype_Int16_key_24_iv_24 = r"""'1D52'""" + +example_mode_aes_128_cfb1_datatype_Int32_key_24_iv_24 = r"""'1D52BAD5'""" + +example_mode_aes_128_cfb1_datatype_Int64_key_24_iv_24 = r"""'1D52BAD5E209AC18'""" + +example_mode_aes_128_cfb1_datatype_Float32_key_24_iv_24 = r"""'1C3165A0'""" + +example_mode_aes_128_cfb1_datatype_Float64_key_24_iv_24 = r"""'1C31E017FDE1A7AC'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_key_24_iv_24 = r"""'2CCB18B8'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_key_24_iv_24 = r"""'2CCB18B874E51783'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_key_24_iv_24 = r"""'2CCB18B874E51783A6ADE353B5D95026'""" + +example_mode_aes_128_cfb1_datatype_UUID_key_24_iv_24 = r"""'EF014A2E68CFB6C3BE44AE59F4A58888'""" + +example_mode_aes_128_cfb1_datatype_Date_key_24_iv_24 = r"""'70F3'""" + +example_mode_aes_128_cfb1_datatype_DateTime_key_24_iv_24 = r"""'6FF3F637'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_key_24_iv_24 = r"""'2432F8327D29D7FF'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_key_24_iv_24 = r"""'30'""" + +example_mode_aes_128_cfb1_datatype_Array_key_24_iv_24 = r"""'1D50'""" + +example_mode_aes_128_cfb1_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_key_24_iv_24 = r"""'20BA7FDE'""" + +example_mode_aes_128_cfb1_datatype_IPv6_key_24_iv_24 = r"""'2C8F73DFF566693C8E482419FAC24837'""" + +example_mode_aes_128_cfb1_datatype_Enum8_key_24_iv_24 = r"""'1D'""" + +example_mode_aes_128_cfb1_datatype_Enum16_key_24_iv_24 = r"""'1D52'""" + +example_mode_aes_192_cfb1_datatype_bytes_key_24_iv_16 = r"""'07'""" + +example_mode_aes_192_cfb1_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_cfb1_datatype_utf8string_key_24_iv_16 = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_key_24_iv_16 = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_192_cfb1_datatype_String_key_24_iv_16 = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_FixedString_key_24_iv_16 = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_UInt8_key_24_iv_16 = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_UInt16_key_24_iv_16 = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_UInt32_key_24_iv_16 = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_UInt64_key_24_iv_16 = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Int8_key_24_iv_16 = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Int16_key_24_iv_16 = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_Int32_key_24_iv_16 = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_Int64_key_24_iv_16 = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Float32_key_24_iv_16 = r"""'07955BCF'""" + +example_mode_aes_192_cfb1_datatype_Float64_key_24_iv_16 = r"""'0795A57CA222A36E'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_key_24_iv_16 = r"""'2A15BB86'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_key_24_iv_16 = r"""'2A15BB86FB961E7D'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_key_24_iv_16 = r"""'2A15BB86FB961E7D0DD5055987176AF4'""" + +example_mode_aes_192_cfb1_datatype_UUID_key_24_iv_16 = r"""'DA2338793C7B9E0F6722E272062F5EA1'""" + +example_mode_aes_192_cfb1_datatype_Date_key_24_iv_16 = r"""'4AAB'""" + +example_mode_aes_192_cfb1_datatype_DateTime_key_24_iv_16 = r"""'5B6A8EE6'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_key_24_iv_16 = r"""'23C4E2A707F73EF4'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_key_24_iv_16 = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_Array_key_24_iv_16 = r"""'069C'""" + +example_mode_aes_192_cfb1_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_key_24_iv_16 = r"""'2470A839'""" + +example_mode_aes_192_cfb1_datatype_IPv6_key_24_iv_16 = r"""'2A712A746781131B2DC4EB92E31C72FA'""" + +example_mode_aes_192_cfb1_datatype_Enum8_key_24_iv_16 = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Enum16_key_24_iv_16 = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_bytes_key_32_iv_32 = r"""'6A'""" + +example_mode_aes_192_cfb1_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_cfb1_datatype_utf8string_key_32_iv_32 = r"""'1317B03E3411FE499C234D87DD44014E46B15B10C9F76A3C'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_key_32_iv_32 = r"""'1317B03E3411FE499C234D87DD44014E46B15B10C9F76A3C'""" + +example_mode_aes_192_cfb1_datatype_String_key_32_iv_32 = r"""'45'""" + +example_mode_aes_192_cfb1_datatype_FixedString_key_32_iv_32 = r"""'45'""" + +example_mode_aes_192_cfb1_datatype_UInt8_key_32_iv_32 = r"""'6B'""" + +example_mode_aes_192_cfb1_datatype_UInt16_key_32_iv_32 = r"""'6B43'""" + +example_mode_aes_192_cfb1_datatype_UInt32_key_32_iv_32 = r"""'6B43D96E'""" + +example_mode_aes_192_cfb1_datatype_UInt64_key_32_iv_32 = r"""'6B43D96E7D63C177'""" + +example_mode_aes_192_cfb1_datatype_Int8_key_32_iv_32 = r"""'6B'""" + +example_mode_aes_192_cfb1_datatype_Int16_key_32_iv_32 = r"""'6B43'""" + +example_mode_aes_192_cfb1_datatype_Int32_key_32_iv_32 = r"""'6B43D96E'""" + +example_mode_aes_192_cfb1_datatype_Int64_key_32_iv_32 = r"""'6B43D96E7D63C177'""" + +example_mode_aes_192_cfb1_datatype_Float32_key_32_iv_32 = r"""'6AF301CD'""" + +example_mode_aes_192_cfb1_datatype_Float64_key_32_iv_32 = r"""'6AF3B787907E47A9'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_key_32_iv_32 = r"""'515FA59C'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_key_32_iv_32 = r"""'515FA59C52E93CA7'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_key_32_iv_32 = r"""'515FA59C52E93CA74BFFFC3D5F3D3163'""" + +example_mode_aes_192_cfb1_datatype_UUID_key_32_iv_32 = r"""'E5F370149365AFB35B9245146396A914'""" + +example_mode_aes_192_cfb1_datatype_Date_key_32_iv_32 = r"""'0685'""" + +example_mode_aes_192_cfb1_datatype_DateTime_key_32_iv_32 = r"""'1D0054E1'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_key_32_iv_32 = r"""'5BD19E5F525FA135'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_key_32_iv_32 = r"""'45'""" + +example_mode_aes_192_cfb1_datatype_Array_key_32_iv_32 = r"""'6B40'""" + +example_mode_aes_192_cfb1_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_key_32_iv_32 = r"""'5D77DE04'""" + +example_mode_aes_192_cfb1_datatype_IPv6_key_32_iv_32 = r"""'513F29EF9D609D6DBD12084B5E11D796'""" + +example_mode_aes_192_cfb1_datatype_Enum8_key_32_iv_32 = r"""'6B'""" + +example_mode_aes_192_cfb1_datatype_Enum16_key_32_iv_32 = r"""'6B43'""" + +example_mode_aes_256_cfb1_datatype_bytes_key_32_iv_16 = r"""'7F'""" + +example_mode_aes_256_cfb1_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_cfb1_datatype_utf8string_key_32_iv_16 = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_256_cfb1_datatype_String_key_32_iv_16 = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_FixedString_key_32_iv_16 = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_UInt8_key_32_iv_16 = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_UInt16_key_32_iv_16 = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_UInt32_key_32_iv_16 = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_UInt64_key_32_iv_16 = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Int8_key_32_iv_16 = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Int16_key_32_iv_16 = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_Int32_key_32_iv_16 = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_Int64_key_32_iv_16 = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Float32_key_32_iv_16 = r"""'7F630BBA'""" + +example_mode_aes_256_cfb1_datatype_Float64_key_32_iv_16 = r"""'7F638DFAAA434E6B'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_key_32_iv_16 = r"""'4F430FBA'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_key_32_iv_16 = r"""'4F430FBAA3AAF884'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_key_32_iv_16 = r"""'4F430FBAA3AAF8845DB7BBA7F98F49C4'""" + +example_mode_aes_256_cfb1_datatype_UUID_key_32_iv_16 = r"""'B06F4A8C3BF3A8D32D113D0D40397C8F'""" + +example_mode_aes_256_cfb1_datatype_Date_key_32_iv_16 = r"""'30CE'""" + +example_mode_aes_256_cfb1_datatype_DateTime_key_32_iv_16 = r"""'206545FA'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_key_32_iv_16 = r"""'43756F28C68E3D55'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_key_32_iv_16 = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_Array_key_32_iv_16 = r"""'7EA3'""" + +example_mode_aes_256_cfb1_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_key_32_iv_16 = r"""'4526FCCF'""" + +example_mode_aes_256_cfb1_datatype_IPv6_key_32_iv_16 = r"""'4F23BDAC741DB8767CE6AE24888545A2'""" + +example_mode_aes_256_cfb1_datatype_Enum8_key_32_iv_16 = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Enum16_key_32_iv_16 = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_bytes_key_64_iv_64 = r"""'DF'""" + +example_mode_aes_256_cfb1_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_cfb1_datatype_utf8string_key_64_iv_64 = r"""'A72DF5A969944B55E9D33B4E5207FB80DAB5BA6787ACB19F'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_key_64_iv_64 = r"""'A72DF5A969944B55E9D33B4E5207FB80DAB5BA6787ACB19F'""" + +example_mode_aes_256_cfb1_datatype_String_key_64_iv_64 = r"""'E9'""" + +example_mode_aes_256_cfb1_datatype_FixedString_key_64_iv_64 = r"""'E9'""" + +example_mode_aes_256_cfb1_datatype_UInt8_key_64_iv_64 = r"""'DE'""" + +example_mode_aes_256_cfb1_datatype_UInt16_key_64_iv_64 = r"""'DEBD'""" + +example_mode_aes_256_cfb1_datatype_UInt32_key_64_iv_64 = r"""'DEBD9C91'""" + +example_mode_aes_256_cfb1_datatype_UInt64_key_64_iv_64 = r"""'DEBD9C915CAA4813'""" + +example_mode_aes_256_cfb1_datatype_Int8_key_64_iv_64 = r"""'DE'""" + +example_mode_aes_256_cfb1_datatype_Int16_key_64_iv_64 = r"""'DEBD'""" + +example_mode_aes_256_cfb1_datatype_Int32_key_64_iv_64 = r"""'DEBD9C91'""" + +example_mode_aes_256_cfb1_datatype_Int64_key_64_iv_64 = r"""'DEBD9C915CAA4813'""" + +example_mode_aes_256_cfb1_datatype_Float32_key_64_iv_64 = r"""'DF845158'""" + +example_mode_aes_256_cfb1_datatype_Float64_key_64_iv_64 = r"""'DF84903EDD435363'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_key_64_iv_64 = r"""'F2FE11DD'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_key_64_iv_64 = r"""'F2FE11DDDF1B0FB8'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_key_64_iv_64 = r"""'F2FE11DDDF1B0FB8F5BA32A48936DF25'""" + +example_mode_aes_256_cfb1_datatype_UUID_key_64_iv_64 = r"""'580073A89DF8ED751F3E63D07B5AAE93'""" + +example_mode_aes_256_cfb1_datatype_Date_key_64_iv_64 = r"""'BCC3'""" + +example_mode_aes_256_cfb1_datatype_DateTime_key_64_iv_64 = r"""'AF6825D1'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_key_64_iv_64 = r"""'FD4A027DF5DD3405'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_key_64_iv_64 = r"""'E9'""" + +example_mode_aes_256_cfb1_datatype_Array_key_64_iv_64 = r"""'DEBE'""" + +example_mode_aes_256_cfb1_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_key_64_iv_64 = r"""'FB2ABD13'""" + +example_mode_aes_256_cfb1_datatype_IPv6_key_64_iv_64 = r"""'F2B6BE1E1DC716AD0490FF91D509F72B'""" + +example_mode_aes_256_cfb1_datatype_Enum8_key_64_iv_64 = r"""'DE'""" + +example_mode_aes_256_cfb1_datatype_Enum16_key_64_iv_64 = r"""'DEBD'""" + +example_mode_aes_128_cfb8_datatype_bytes_key_16_iv_None = r"""'10'""" + +example_mode_aes_128_cfb8_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_cfb8_datatype_utf8string_key_16_iv_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_key_16_iv_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_128_cfb8_datatype_String_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_FixedString_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_UInt8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_UInt16_key_16_iv_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_UInt32_key_16_iv_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_UInt64_key_16_iv_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Int8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Int16_key_16_iv_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_Int32_key_16_iv_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_Int64_key_16_iv_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Float32_key_16_iv_None = r"""'10671940'""" + +example_mode_aes_128_cfb8_datatype_Float64_key_16_iv_None = r"""'106799607DBF56DA'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_key_16_iv_None = r"""'30756C94'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_key_16_iv_None = r"""'30756C9417D3C023'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_key_16_iv_None = r"""'30756C9417D3C023705550B7BEF872FF'""" + +example_mode_aes_128_cfb8_datatype_UUID_key_16_iv_None = r"""'F7FE50CF0647659CB0D401B5C0E259D3'""" + +example_mode_aes_128_cfb8_datatype_Date_key_16_iv_None = r"""'46EA'""" + +example_mode_aes_128_cfb8_datatype_DateTime_key_16_iv_None = r"""'5EB4905E'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_key_16_iv_None = r"""'3BB70F8E64D7C6A7'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_Array_key_16_iv_None = r"""'11FD'""" + +example_mode_aes_128_cfb8_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_key_16_iv_None = r"""'3DC2BE9E'""" + +example_mode_aes_128_cfb8_datatype_IPv6_key_16_iv_None = r"""'303ABAC6F4F380D9F077DFC79C82D1A1'""" + +example_mode_aes_128_cfb8_datatype_Enum8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Enum16_key_16_iv_None = r"""'11FF'""" + +example_mode_aes_192_cfb8_datatype_bytes_key_24_iv_None = r"""'07'""" + +example_mode_aes_192_cfb8_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_cfb8_datatype_utf8string_key_24_iv_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_key_24_iv_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_192_cfb8_datatype_String_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_FixedString_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_UInt8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_UInt16_key_24_iv_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_UInt32_key_24_iv_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_UInt64_key_24_iv_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Int8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Int16_key_24_iv_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_Int32_key_24_iv_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_Int64_key_24_iv_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Float32_key_24_iv_None = r"""'07EDB300'""" + +example_mode_aes_192_cfb8_datatype_Float64_key_24_iv_None = r"""'07ED3359B91DEF3B'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_key_24_iv_None = r"""'275947FE'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_key_24_iv_None = r"""'275947FE4B3390EE'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_key_24_iv_None = r"""'275947FE4B3390EE7A2541BC8E2F58D7'""" + +example_mode_aes_192_cfb8_datatype_UUID_key_24_iv_None = r"""'E0C082C032FB8ED756F9345E270A283B'""" + +example_mode_aes_192_cfb8_datatype_Date_key_24_iv_None = r"""'5109'""" + +example_mode_aes_192_cfb8_datatype_DateTime_key_24_iv_None = r"""'49713150'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_key_24_iv_None = r"""'2C10FB4FEC471EF7'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_Array_key_24_iv_None = r"""'0681'""" + +example_mode_aes_192_cfb8_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_key_24_iv_None = r"""'2A41C8F2'""" + +example_mode_aes_192_cfb8_datatype_IPv6_key_24_iv_None = r"""'271682C9379C5A46C68488DC33D0C278'""" + +example_mode_aes_192_cfb8_datatype_Enum8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Enum16_key_24_iv_None = r"""'0683'""" + +example_mode_aes_256_cfb8_datatype_bytes_key_32_iv_None = r"""'B0'""" + +example_mode_aes_256_cfb8_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_cfb8_datatype_utf8string_key_32_iv_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_key_32_iv_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_256_cfb8_datatype_String_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_FixedString_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_UInt8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_UInt16_key_32_iv_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_UInt32_key_32_iv_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_UInt64_key_32_iv_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Int8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Int16_key_32_iv_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_Int32_key_32_iv_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_Int64_key_32_iv_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Float32_key_32_iv_None = r"""'B05A05BE'""" + +example_mode_aes_256_cfb8_datatype_Float64_key_32_iv_None = r"""'B05A8510DB2F16A0'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_key_32_iv_None = r"""'906B5777'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_key_32_iv_None = r"""'906B57771CB81F37'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_key_32_iv_None = r"""'906B57771CB81F378D932AE788527DE2'""" + +example_mode_aes_256_cfb8_datatype_UUID_key_32_iv_None = r"""'57FB06BA6F4BA51D7A61D65A7827A18D'""" + +example_mode_aes_256_cfb8_datatype_Date_key_32_iv_None = r"""'E652'""" + +example_mode_aes_256_cfb8_datatype_DateTime_key_32_iv_None = r"""'FEEEADA4'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_key_32_iv_None = r"""'9BB36DEF05FF5975'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_Array_key_32_iv_None = r"""'B15D'""" + +example_mode_aes_256_cfb8_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_key_32_iv_None = r"""'9DC836F3'""" + +example_mode_aes_256_cfb8_datatype_IPv6_key_32_iv_None = r"""'90242F0083C8B0221DF3B5755EC8D99C'""" + +example_mode_aes_256_cfb8_datatype_Enum8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Enum16_key_32_iv_None = r"""'B15F'""" + +example_mode_aes_128_cfb8_datatype_bytes_key_16_iv_16 = r"""'32'""" + +example_mode_aes_128_cfb8_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_cfb8_datatype_utf8string_key_16_iv_16 = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_key_16_iv_16 = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_128_cfb8_datatype_String_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_FixedString_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_UInt8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_UInt16_key_16_iv_16 = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_UInt32_key_16_iv_16 = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_UInt64_key_16_iv_16 = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Int8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Int16_key_16_iv_16 = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_Int32_key_16_iv_16 = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_Int64_key_16_iv_16 = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Float32_key_16_iv_16 = r"""'3232B23D'""" + +example_mode_aes_128_cfb8_datatype_Float64_key_16_iv_16 = r"""'323232323232C2A6'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_key_16_iv_16 = r"""'12ABA873'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_key_16_iv_16 = r"""'12ABA873E2E24473'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_key_16_iv_16 = r"""'12ABA873E2E24473166434D82270A19C'""" + +example_mode_aes_128_cfb8_datatype_UUID_key_16_iv_16 = r"""'D529D970A38CCB794F856E4458D0E2D4'""" + +example_mode_aes_128_cfb8_datatype_Date_key_16_iv_16 = r"""'6445'""" + +example_mode_aes_128_cfb8_datatype_DateTime_key_16_iv_16 = r"""'7CBF2FDA'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_key_16_iv_16 = r"""'191C7B5A063F562D'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_Array_key_16_iv_16 = r"""'336A'""" + +example_mode_aes_128_cfb8_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_key_16_iv_16 = r"""'1F0A367A'""" + +example_mode_aes_128_cfb8_datatype_IPv6_key_16_iv_16 = r"""'12E4B19D97DC9F2C61A671C51B1201D2'""" + +example_mode_aes_128_cfb8_datatype_Enum8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Enum16_key_16_iv_16 = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_bytes_key_24_iv_24 = r"""'5B'""" + +example_mode_aes_128_cfb8_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_cfb8_datatype_utf8string_key_24_iv_24 = r"""'1CE086607CFD8523D699E2588307E8F47EB1C6DB33141C6A'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_key_24_iv_24 = r"""'1CE086607CFD8523D699E2588307E8F47EB1C6DB33141C6A'""" + +example_mode_aes_128_cfb8_datatype_String_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb8_datatype_FixedString_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb8_datatype_UInt8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb8_datatype_UInt16_key_24_iv_24 = r"""'5A37'""" + +example_mode_aes_128_cfb8_datatype_UInt32_key_24_iv_24 = r"""'5A37BD02'""" + +example_mode_aes_128_cfb8_datatype_UInt64_key_24_iv_24 = r"""'5A37BD02ECFA56DB'""" + +example_mode_aes_128_cfb8_datatype_Int8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb8_datatype_Int16_key_24_iv_24 = r"""'5A37'""" + +example_mode_aes_128_cfb8_datatype_Int32_key_24_iv_24 = r"""'5A37BD02'""" + +example_mode_aes_128_cfb8_datatype_Int64_key_24_iv_24 = r"""'5A37BD02ECFA56DB'""" + +example_mode_aes_128_cfb8_datatype_Float32_key_24_iv_24 = r"""'5BC333D3'""" + +example_mode_aes_128_cfb8_datatype_Float64_key_24_iv_24 = r"""'5BC3B3495464BBE8'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_key_24_iv_24 = r"""'7BE3C6D3'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_key_24_iv_24 = r"""'7BE3C6D356A31F6E'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_key_24_iv_24 = r"""'7BE3C6D356A31F6E1A17102C88C40C85'""" + +example_mode_aes_128_cfb8_datatype_UUID_key_24_iv_24 = r"""'BC30ED9E33095BED0825C18860C622AC'""" + +example_mode_aes_128_cfb8_datatype_Date_key_24_iv_24 = r"""'0D03'""" + +example_mode_aes_128_cfb8_datatype_DateTime_key_24_iv_24 = r"""'15F76197'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_key_24_iv_24 = r"""'7035C3E2D0F5F702'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb8_datatype_Array_key_24_iv_24 = r"""'5A35'""" + +example_mode_aes_128_cfb8_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_key_24_iv_24 = r"""'769533D5'""" + +example_mode_aes_128_cfb8_datatype_IPv6_key_24_iv_24 = r"""'7BAC6FE32B92AB3A07AA8D90F0E175BA'""" + +example_mode_aes_128_cfb8_datatype_Enum8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb8_datatype_Enum16_key_24_iv_24 = r"""'5A37'""" + +example_mode_aes_192_cfb8_datatype_bytes_key_24_iv_16 = r"""'68'""" + +example_mode_aes_192_cfb8_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_cfb8_datatype_utf8string_key_24_iv_16 = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_key_24_iv_16 = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_192_cfb8_datatype_String_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_FixedString_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_UInt8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_UInt16_key_24_iv_16 = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_UInt32_key_24_iv_16 = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_UInt64_key_24_iv_16 = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Int8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Int16_key_24_iv_16 = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_Int32_key_24_iv_16 = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_Int64_key_24_iv_16 = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Float32_key_24_iv_16 = r"""'6861DF7A'""" + +example_mode_aes_192_cfb8_datatype_Float64_key_24_iv_16 = r"""'68615FBC184B8D50'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_key_24_iv_16 = r"""'48041B5C'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_key_24_iv_16 = r"""'48041B5C6BEF70DD'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_key_24_iv_16 = r"""'48041B5C6BEF70DD4CDABC1FC8C2C684'""" + +example_mode_aes_192_cfb8_datatype_UUID_key_24_iv_16 = r"""'8FF1142976A9808C0F475A3D2A34D06D'""" + +example_mode_aes_192_cfb8_datatype_Date_key_24_iv_16 = r"""'3E6D'""" + +example_mode_aes_192_cfb8_datatype_DateTime_key_24_iv_16 = r"""'269AFDC7'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_key_24_iv_16 = r"""'4350703E05F43A50'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_Array_key_24_iv_16 = r"""'6926'""" + +example_mode_aes_192_cfb8_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_key_24_iv_16 = r"""'45979A4C'""" + +example_mode_aes_192_cfb8_datatype_IPv6_key_24_iv_16 = r"""'484BFA49756D837181B7EE03EBCF2B79'""" + +example_mode_aes_192_cfb8_datatype_Enum8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Enum16_key_24_iv_16 = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_bytes_key_32_iv_32 = r"""'31'""" + +example_mode_aes_192_cfb8_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_cfb8_datatype_utf8string_key_32_iv_32 = r"""'7625D50FBB8F39E353302002F643A769A00CA77A81D316AB'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_key_32_iv_32 = r"""'7625D50FBB8F39E353302002F643A769A00CA77A81D316AB'""" + +example_mode_aes_192_cfb8_datatype_String_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb8_datatype_FixedString_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb8_datatype_UInt8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb8_datatype_UInt16_key_32_iv_32 = r"""'3032'""" + +example_mode_aes_192_cfb8_datatype_UInt32_key_32_iv_32 = r"""'30325F8E'""" + +example_mode_aes_192_cfb8_datatype_UInt64_key_32_iv_32 = r"""'30325F8E9BE4C241'""" + +example_mode_aes_192_cfb8_datatype_Int8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb8_datatype_Int16_key_32_iv_32 = r"""'3032'""" + +example_mode_aes_192_cfb8_datatype_Int32_key_32_iv_32 = r"""'30325F8E'""" + +example_mode_aes_192_cfb8_datatype_Int64_key_32_iv_32 = r"""'30325F8E9BE4C241'""" + +example_mode_aes_192_cfb8_datatype_Float32_key_32_iv_32 = r"""'3132C141'""" + +example_mode_aes_192_cfb8_datatype_Float64_key_32_iv_32 = r"""'3132410E0E8B4763'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_key_32_iv_32 = r"""'1143551C'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_key_32_iv_32 = r"""'1143551CF390615B'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_key_32_iv_32 = r"""'1143551CF390615BB25617FAD1010676'""" + +example_mode_aes_192_cfb8_datatype_UUID_key_32_iv_32 = r"""'D63F190E24A3EF3805EF4E394CD5C620'""" + +example_mode_aes_192_cfb8_datatype_Date_key_32_iv_32 = r"""'67FC'""" + +example_mode_aes_192_cfb8_datatype_DateTime_key_32_iv_32 = r"""'7FDB8ABC'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_key_32_iv_32 = r"""'1A304FE168BFC721'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb8_datatype_Array_key_32_iv_32 = r"""'3030'""" + +example_mode_aes_192_cfb8_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_key_32_iv_32 = r"""'1CE26C68'""" + +example_mode_aes_192_cfb8_datatype_IPv6_key_32_iv_32 = r"""'110CAE7AAE3AF3436C8D155FD4DFACB9'""" + +example_mode_aes_192_cfb8_datatype_Enum8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb8_datatype_Enum16_key_32_iv_32 = r"""'3032'""" + +example_mode_aes_256_cfb8_datatype_bytes_key_32_iv_16 = r"""'69'""" + +example_mode_aes_256_cfb8_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_cfb8_datatype_utf8string_key_32_iv_16 = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_256_cfb8_datatype_String_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_FixedString_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_UInt8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_UInt16_key_32_iv_16 = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_UInt32_key_32_iv_16 = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_UInt64_key_32_iv_16 = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Int8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Int16_key_32_iv_16 = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_Int32_key_32_iv_16 = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_Int64_key_32_iv_16 = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Float32_key_32_iv_16 = r"""'69B127F9'""" + +example_mode_aes_256_cfb8_datatype_Float64_key_32_iv_16 = r"""'69B1A72CB81A0FFF'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_key_32_iv_16 = r"""'49378750'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_key_32_iv_16 = r"""'493787505DFF5606'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_key_32_iv_16 = r"""'493787505DFF5606774649C631E6E0E7'""" + +example_mode_aes_256_cfb8_datatype_UUID_key_32_iv_16 = r"""'8E09A60AA21565C888B2D92942896930'""" + +example_mode_aes_256_cfb8_datatype_Date_key_32_iv_16 = r"""'3FF1'""" + +example_mode_aes_256_cfb8_datatype_DateTime_key_32_iv_16 = r"""'274E13D8'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_key_32_iv_16 = r"""'4211DFF611769F37'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_Array_key_32_iv_16 = r"""'682E'""" + +example_mode_aes_256_cfb8_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_key_32_iv_16 = r"""'442DB771'""" + +example_mode_aes_256_cfb8_datatype_IPv6_key_32_iv_16 = r"""'4978AF3EED91F4AD14F7C326CCD96804'""" + +example_mode_aes_256_cfb8_datatype_Enum8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Enum16_key_32_iv_16 = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_bytes_key_64_iv_64 = r"""'D3'""" + +example_mode_aes_256_cfb8_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_cfb8_datatype_utf8string_key_64_iv_64 = r"""'94336E21D7AF28FAA2179962917236281AB6CA2EB17A5F52'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_key_64_iv_64 = r"""'94336E21D7AF28FAA2179962917236281AB6CA2EB17A5F52'""" + +example_mode_aes_256_cfb8_datatype_String_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb8_datatype_FixedString_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb8_datatype_UInt8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb8_datatype_UInt16_key_64_iv_64 = r"""'D2F0'""" + +example_mode_aes_256_cfb8_datatype_UInt32_key_64_iv_64 = r"""'D2F009AE'""" + +example_mode_aes_256_cfb8_datatype_UInt64_key_64_iv_64 = r"""'D2F009AE234CE3B2'""" + +example_mode_aes_256_cfb8_datatype_Int8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb8_datatype_Int16_key_64_iv_64 = r"""'D2F0'""" + +example_mode_aes_256_cfb8_datatype_Int32_key_64_iv_64 = r"""'D2F009AE'""" + +example_mode_aes_256_cfb8_datatype_Int64_key_64_iv_64 = r"""'D2F009AE234CE3B2'""" + +example_mode_aes_256_cfb8_datatype_Float32_key_64_iv_64 = r"""'D323A6EE'""" + +example_mode_aes_256_cfb8_datatype_Float64_key_64_iv_64 = r"""'D32326952E7D73B6'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_key_64_iv_64 = r"""'F3600DB1'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_key_64_iv_64 = r"""'F3600DB150D7A5EF'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_key_64_iv_64 = r"""'F3600DB150D7A5EFA1934028A5BDD856'""" + +example_mode_aes_256_cfb8_datatype_UUID_key_64_iv_64 = r"""'34B0729B2D7A5D08043DBF031FE464BC'""" + +example_mode_aes_256_cfb8_datatype_Date_key_64_iv_64 = r"""'85A9'""" + +example_mode_aes_256_cfb8_datatype_DateTime_key_64_iv_64 = r"""'9D298352'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_key_64_iv_64 = r"""'F8BC2B190E2139CC'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb8_datatype_Array_key_64_iv_64 = r"""'D2F2'""" + +example_mode_aes_256_cfb8_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_key_64_iv_64 = r"""'FEEA2143'""" + +example_mode_aes_256_cfb8_datatype_IPv6_key_64_iv_64 = r"""'F32F6C4967EF6EFF9975263D9074D4E1'""" + +example_mode_aes_256_cfb8_datatype_Enum8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb8_datatype_Enum16_key_64_iv_64 = r"""'D2F0'""" + +example_mode_aes_128_cfb128_datatype_bytes_key_16_iv_None = r"""'10'""" + +example_mode_aes_128_cfb128_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_cfb128_datatype_utf8string_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_128_cfb128_datatype_String_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_FixedString_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_UInt8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_UInt16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_UInt32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_UInt64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Int8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Int16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_Int32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_Int64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Float32_key_16_iv_None = r"""'10DF418A'""" + +example_mode_aes_128_cfb128_datatype_Float64_key_16_iv_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_key_16_iv_None = r"""'3091C1B5'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_key_16_iv_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_key_16_iv_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_cfb128_datatype_UUID_key_16_iv_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_cfb128_datatype_Date_key_16_iv_None = r"""'4698'""" + +example_mode_aes_128_cfb128_datatype_DateTime_key_16_iv_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_key_16_iv_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_Array_key_16_iv_None = r"""'11DD'""" + +example_mode_aes_128_cfb128_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_key_16_iv_None = r"""'3D5D201E'""" + +example_mode_aes_128_cfb128_datatype_IPv6_key_16_iv_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_cfb128_datatype_Enum8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Enum16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_192_cfb128_datatype_bytes_key_24_iv_None = r"""'07'""" + +example_mode_aes_192_cfb128_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_cfb128_datatype_utf8string_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_192_cfb128_datatype_String_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_FixedString_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_UInt8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_UInt16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_UInt64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Int8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Int16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_Int32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_Int64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Float32_key_24_iv_None = r"""'07B799A2'""" + +example_mode_aes_192_cfb128_datatype_Float64_key_24_iv_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_key_24_iv_None = r"""'27F9199D'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_key_24_iv_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_key_24_iv_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_cfb128_datatype_UUID_key_24_iv_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_cfb128_datatype_Date_key_24_iv_None = r"""'51F0'""" + +example_mode_aes_192_cfb128_datatype_DateTime_key_24_iv_None = r"""'496715C3'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_key_24_iv_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_Array_key_24_iv_None = r"""'06B5'""" + +example_mode_aes_192_cfb128_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_key_24_iv_None = r"""'2A35F836'""" + +example_mode_aes_192_cfb128_datatype_IPv6_key_24_iv_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_cfb128_datatype_Enum8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Enum16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_256_cfb128_datatype_bytes_key_32_iv_None = r"""'B0'""" + +example_mode_aes_256_cfb128_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_cfb128_datatype_utf8string_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_256_cfb128_datatype_String_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_FixedString_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_UInt8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_UInt16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_UInt32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_UInt64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Int8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Int16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_Int32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_Int64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Float32_key_32_iv_None = r"""'B08E4FA1'""" + +example_mode_aes_256_cfb128_datatype_Float64_key_32_iv_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_key_32_iv_None = r"""'90C0CF9E'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_key_32_iv_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_key_32_iv_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_cfb128_datatype_UUID_key_32_iv_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_cfb128_datatype_Date_key_32_iv_None = r"""'E6C9'""" + +example_mode_aes_256_cfb128_datatype_DateTime_key_32_iv_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_key_32_iv_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_Array_key_32_iv_None = r"""'B18C'""" + +example_mode_aes_256_cfb128_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_key_32_iv_None = r"""'9D0C2E35'""" + +example_mode_aes_256_cfb128_datatype_IPv6_key_32_iv_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_cfb128_datatype_Enum8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Enum16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_128_cfb128_datatype_bytes_key_16_iv_16 = r"""'32'""" + +example_mode_aes_128_cfb128_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_cfb128_datatype_utf8string_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_128_cfb128_datatype_String_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_FixedString_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_UInt8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_UInt16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_UInt32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_UInt64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Int8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Int16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_Int32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_Int64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Float32_key_16_iv_16 = r"""'328829BB'""" + +example_mode_aes_128_cfb128_datatype_Float64_key_16_iv_16 = r"""'3288A984DD060F67'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_key_16_iv_16 = r"""'12C6A984'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_key_16_iv_16 = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_key_16_iv_16 = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_cfb128_datatype_UUID_key_16_iv_16 = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_cfb128_datatype_Date_key_16_iv_16 = r"""'64CF'""" + +example_mode_aes_128_cfb128_datatype_DateTime_key_16_iv_16 = r"""'7C58A5DA'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_key_16_iv_16 = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_Array_key_16_iv_16 = r"""'338A'""" + +example_mode_aes_128_cfb128_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_key_16_iv_16 = r"""'1F0A482F'""" + +example_mode_aes_128_cfb128_datatype_IPv6_key_16_iv_16 = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_cfb128_datatype_Enum8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Enum16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_bytes_key_24_iv_24 = r"""'5B'""" + +example_mode_aes_128_cfb128_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_cfb128_datatype_utf8string_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5C6FFFA5BE700B5B9B'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5C6FFFA5BE700B5B9B'""" + +example_mode_aes_128_cfb128_datatype_String_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb128_datatype_FixedString_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb128_datatype_UInt8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb128_datatype_UInt16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_cfb128_datatype_UInt32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_cfb128_datatype_UInt64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_cfb128_datatype_Int8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb128_datatype_Int16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_cfb128_datatype_Int32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_cfb128_datatype_Int64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_cfb128_datatype_Float32_key_24_iv_24 = r"""'5B7782CA'""" + +example_mode_aes_128_cfb128_datatype_Float64_key_24_iv_24 = r"""'5B7702F5E9B07D88'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_key_24_iv_24 = r"""'7B3902F5'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_key_24_iv_24 = r"""'7B3902F5E9B08DB7'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_key_24_iv_24 = r"""'7B3902F5E9B08DB73E1BAC7CCCCDAEF6'""" + +example_mode_aes_128_cfb128_datatype_UUID_key_24_iv_24 = r"""'BC66B1A9ED747DD69EC07F16CC6BD566'""" + +example_mode_aes_128_cfb128_datatype_Date_key_24_iv_24 = r"""'0D30'""" + +example_mode_aes_128_cfb128_datatype_DateTime_key_24_iv_24 = r"""'15A70EAB'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_key_24_iv_24 = r"""'70C60F9786B18DB7'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb128_datatype_Array_key_24_iv_24 = r"""'5A75'""" + +example_mode_aes_128_cfb128_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_key_24_iv_24 = r"""'76F5E35E'""" + +example_mode_aes_128_cfb128_datatype_IPv6_key_24_iv_24 = r"""'7B760F4DE9B008143E1BAC7C60D22EF7'""" + +example_mode_aes_128_cfb128_datatype_Enum8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb128_datatype_Enum16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_192_cfb128_datatype_bytes_key_24_iv_16 = r"""'68'""" + +example_mode_aes_192_cfb128_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_cfb128_datatype_utf8string_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_192_cfb128_datatype_String_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_FixedString_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_UInt8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_UInt16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_UInt64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Int8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Int16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_Int32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_Int64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Float32_key_24_iv_16 = r"""'68C767AD'""" + +example_mode_aes_192_cfb128_datatype_Float64_key_24_iv_16 = r"""'68C7E792B710878E'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_key_24_iv_16 = r"""'4889E792'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_key_24_iv_16 = r"""'4889E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_key_24_iv_16 = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_cfb128_datatype_UUID_key_24_iv_16 = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_cfb128_datatype_Date_key_24_iv_16 = r"""'3E80'""" + +example_mode_aes_192_cfb128_datatype_DateTime_key_24_iv_16 = r"""'2617EBCC'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_key_24_iv_16 = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_Array_key_24_iv_16 = r"""'69C5'""" + +example_mode_aes_192_cfb128_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_key_24_iv_16 = r"""'45450639'""" + +example_mode_aes_192_cfb128_datatype_IPv6_key_24_iv_16 = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_cfb128_datatype_Enum8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Enum16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_bytes_key_32_iv_32 = r"""'31'""" + +example_mode_aes_192_cfb128_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_cfb128_datatype_utf8string_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D8D9CFB425308D13E0'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D8D9CFB425308D13E0'""" + +example_mode_aes_192_cfb128_datatype_String_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb128_datatype_FixedString_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb128_datatype_UInt8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb128_datatype_UInt16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_cfb128_datatype_UInt32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_cfb128_datatype_UInt64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_cfb128_datatype_Int8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb128_datatype_Int16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_cfb128_datatype_Int32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_cfb128_datatype_Int64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_cfb128_datatype_Float32_key_32_iv_32 = r"""'31A00E4B'""" + +example_mode_aes_192_cfb128_datatype_Float64_key_32_iv_32 = r"""'31A08E74AFABC0FB'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_key_32_iv_32 = r"""'11EE8E74'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_key_32_iv_32 = r"""'11EE8E74AFAB30C4'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_key_32_iv_32 = r"""'11EE8E74AFAB30C4422B6D36C4A76572'""" + +example_mode_aes_192_cfb128_datatype_UUID_key_32_iv_32 = r"""'D6B13D28AB6FC0A5E2F0BE5CC4011EE2'""" + +example_mode_aes_192_cfb128_datatype_Date_key_32_iv_32 = r"""'67E7'""" + +example_mode_aes_192_cfb128_datatype_DateTime_key_32_iv_32 = r"""'7F70822A'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_key_32_iv_32 = r"""'1A118316C0AA30C4'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb128_datatype_Array_key_32_iv_32 = r"""'30A2'""" + +example_mode_aes_192_cfb128_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_key_32_iv_32 = r"""'1C226FDF'""" + +example_mode_aes_192_cfb128_datatype_IPv6_key_32_iv_32 = r"""'11A183CCAFABB567422B6D3668B8E573'""" + +example_mode_aes_192_cfb128_datatype_Enum8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb128_datatype_Enum16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_256_cfb128_datatype_bytes_key_32_iv_16 = r"""'69'""" + +example_mode_aes_256_cfb128_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_cfb128_datatype_utf8string_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_256_cfb128_datatype_String_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_FixedString_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_UInt8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_UInt16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_UInt32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_UInt64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Int8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Int16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_Int32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_Int64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Float32_key_32_iv_16 = r"""'69580828'""" + +example_mode_aes_256_cfb128_datatype_Float64_key_32_iv_16 = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_key_32_iv_16 = r"""'49168817'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_key_32_iv_16 = r"""'491688173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_key_32_iv_16 = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_cfb128_datatype_UUID_key_32_iv_16 = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_cfb128_datatype_Date_key_32_iv_16 = r"""'3F1F'""" + +example_mode_aes_256_cfb128_datatype_DateTime_key_32_iv_16 = r"""'27888449'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_key_32_iv_16 = r"""'42E9857553DF4488'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_Array_key_32_iv_16 = r"""'685A'""" + +example_mode_aes_256_cfb128_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_key_32_iv_16 = r"""'44DA69BC'""" + +example_mode_aes_256_cfb128_datatype_IPv6_key_32_iv_16 = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_cfb128_datatype_Enum8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Enum16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_bytes_key_64_iv_64 = r"""'D3'""" + +example_mode_aes_256_cfb128_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_cfb128_datatype_utf8string_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D9EC333A5AE718F82'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D9EC333A5AE718F82'""" + +example_mode_aes_256_cfb128_datatype_String_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb128_datatype_FixedString_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb128_datatype_UInt8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb128_datatype_UInt16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_cfb128_datatype_UInt32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_cfb128_datatype_UInt64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_cfb128_datatype_Int8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb128_datatype_Int16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_cfb128_datatype_Int32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_cfb128_datatype_Int64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_cfb128_datatype_Float32_key_64_iv_64 = r"""'D3EE4F63'""" + +example_mode_aes_256_cfb128_datatype_Float64_key_64_iv_64 = r"""'D3EECF5CBB79D542'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_key_64_iv_64 = r"""'F3A0CF5C'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_key_64_iv_64 = r"""'F3A0CF5CBB79257D'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_key_64_iv_64 = r"""'F3A0CF5CBB79257DB12F3A9075610DD7'""" + +example_mode_aes_256_cfb128_datatype_UUID_key_64_iv_64 = r"""'34FF7C00BFBDD51C11F4E9FA75C77647'""" + +example_mode_aes_256_cfb128_datatype_Date_key_64_iv_64 = r"""'85A9'""" + +example_mode_aes_256_cfb128_datatype_DateTime_key_64_iv_64 = r"""'9D3EC302'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_key_64_iv_64 = r"""'F85FC23ED478257D'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb128_datatype_Array_key_64_iv_64 = r"""'D2EC'""" + +example_mode_aes_256_cfb128_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_key_64_iv_64 = r"""'FE6C2EF7'""" + +example_mode_aes_256_cfb128_datatype_IPv6_key_64_iv_64 = r"""'F3EFC2E4BB79A0DEB12F3A90D97E8DD6'""" + +example_mode_aes_256_cfb128_datatype_Enum8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb128_datatype_Enum16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_128_ofb_datatype_bytes_key_16_iv_None = r"""'10'""" + +example_mode_aes_128_ofb_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_ofb_datatype_utf8string_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_128_ofb_datatype_String_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_FixedString_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_UInt8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_UInt16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_UInt32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_UInt64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Int8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Int16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_Int32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_Int64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Float32_key_16_iv_None = r"""'10DF418A'""" + +example_mode_aes_128_ofb_datatype_Float64_key_16_iv_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_ofb_datatype_Decimal32_key_16_iv_None = r"""'3091C1B5'""" + +example_mode_aes_128_ofb_datatype_Decimal64_key_16_iv_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Decimal128_key_16_iv_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_ofb_datatype_UUID_key_16_iv_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_ofb_datatype_Date_key_16_iv_None = r"""'4698'""" + +example_mode_aes_128_ofb_datatype_DateTime_key_16_iv_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_ofb_datatype_DateTime64_key_16_iv_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_Array_key_16_iv_None = r"""'11DD'""" + +example_mode_aes_128_ofb_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_key_16_iv_None = r"""'3D5D201E'""" + +example_mode_aes_128_ofb_datatype_IPv6_key_16_iv_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_ofb_datatype_Enum8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Enum16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_192_ofb_datatype_bytes_key_24_iv_None = r"""'07'""" + +example_mode_aes_192_ofb_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_ofb_datatype_utf8string_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_192_ofb_datatype_String_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_FixedString_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_UInt8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_UInt16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_UInt32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_UInt64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Int8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Int16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_Int32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_Int64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Float32_key_24_iv_None = r"""'07B799A2'""" + +example_mode_aes_192_ofb_datatype_Float64_key_24_iv_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_ofb_datatype_Decimal32_key_24_iv_None = r"""'27F9199D'""" + +example_mode_aes_192_ofb_datatype_Decimal64_key_24_iv_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Decimal128_key_24_iv_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_ofb_datatype_UUID_key_24_iv_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_ofb_datatype_Date_key_24_iv_None = r"""'51F0'""" + +example_mode_aes_192_ofb_datatype_DateTime_key_24_iv_None = r"""'496715C3'""" + +example_mode_aes_192_ofb_datatype_DateTime64_key_24_iv_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_Array_key_24_iv_None = r"""'06B5'""" + +example_mode_aes_192_ofb_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_key_24_iv_None = r"""'2A35F836'""" + +example_mode_aes_192_ofb_datatype_IPv6_key_24_iv_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_ofb_datatype_Enum8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Enum16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_256_ofb_datatype_bytes_key_32_iv_None = r"""'B0'""" + +example_mode_aes_256_ofb_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_ofb_datatype_utf8string_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_256_ofb_datatype_String_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_FixedString_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_UInt8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_UInt16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_UInt32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_UInt64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Int8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Int16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_Int32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_Int64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Float32_key_32_iv_None = r"""'B08E4FA1'""" + +example_mode_aes_256_ofb_datatype_Float64_key_32_iv_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_ofb_datatype_Decimal32_key_32_iv_None = r"""'90C0CF9E'""" + +example_mode_aes_256_ofb_datatype_Decimal64_key_32_iv_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Decimal128_key_32_iv_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_ofb_datatype_UUID_key_32_iv_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_ofb_datatype_Date_key_32_iv_None = r"""'E6C9'""" + +example_mode_aes_256_ofb_datatype_DateTime_key_32_iv_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_ofb_datatype_DateTime64_key_32_iv_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_Array_key_32_iv_None = r"""'B18C'""" + +example_mode_aes_256_ofb_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_key_32_iv_None = r"""'9D0C2E35'""" + +example_mode_aes_256_ofb_datatype_IPv6_key_32_iv_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_ofb_datatype_Enum8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Enum16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_128_ofb_datatype_bytes_key_16_iv_16 = r"""'32'""" + +example_mode_aes_128_ofb_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_ofb_datatype_utf8string_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_128_ofb_datatype_String_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_ofb_datatype_FixedString_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_ofb_datatype_UInt8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_ofb_datatype_UInt16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_UInt32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_UInt64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Int8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Int16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_Int32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_Int64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Float32_key_16_iv_16 = r"""'328829BB'""" + +example_mode_aes_128_ofb_datatype_Float64_key_16_iv_16 = r"""'3288A984DD060F67'""" + +example_mode_aes_128_ofb_datatype_Decimal32_key_16_iv_16 = r"""'12C6A984'""" + +example_mode_aes_128_ofb_datatype_Decimal64_key_16_iv_16 = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Decimal128_key_16_iv_16 = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_ofb_datatype_UUID_key_16_iv_16 = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_ofb_datatype_Date_key_16_iv_16 = r"""'64CF'""" + +example_mode_aes_128_ofb_datatype_DateTime_key_16_iv_16 = r"""'7C58A5DA'""" + +example_mode_aes_128_ofb_datatype_DateTime64_key_16_iv_16 = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_ofb_datatype_Array_key_16_iv_16 = r"""'338A'""" + +example_mode_aes_128_ofb_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_key_16_iv_16 = r"""'1F0A482F'""" + +example_mode_aes_128_ofb_datatype_IPv6_key_16_iv_16 = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_ofb_datatype_Enum8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Enum16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_bytes_key_24_iv_24 = r"""'5B'""" + +example_mode_aes_128_ofb_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_ofb_datatype_utf8string_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5CE691EFBA49066039'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5CE691EFBA49066039'""" + +example_mode_aes_128_ofb_datatype_String_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_ofb_datatype_FixedString_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_ofb_datatype_UInt8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_ofb_datatype_UInt16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_ofb_datatype_UInt32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_ofb_datatype_UInt64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_ofb_datatype_Int8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_ofb_datatype_Int16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_ofb_datatype_Int32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_ofb_datatype_Int64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_ofb_datatype_Float32_key_24_iv_24 = r"""'5B7782CA'""" + +example_mode_aes_128_ofb_datatype_Float64_key_24_iv_24 = r"""'5B7702F5E9B07D88'""" + +example_mode_aes_128_ofb_datatype_Decimal32_key_24_iv_24 = r"""'7B3902F5'""" + +example_mode_aes_128_ofb_datatype_Decimal64_key_24_iv_24 = r"""'7B3902F5E9B08DB7'""" + +example_mode_aes_128_ofb_datatype_Decimal128_key_24_iv_24 = r"""'7B3902F5E9B08DB73E1BAC7CCCCDAEF6'""" + +example_mode_aes_128_ofb_datatype_UUID_key_24_iv_24 = r"""'BC66B1A9ED747DD69EC07F16CC6BD566'""" + +example_mode_aes_128_ofb_datatype_Date_key_24_iv_24 = r"""'0D30'""" + +example_mode_aes_128_ofb_datatype_DateTime_key_24_iv_24 = r"""'15A70EAB'""" + +example_mode_aes_128_ofb_datatype_DateTime64_key_24_iv_24 = r"""'70C60F9786B18DB7'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_ofb_datatype_Array_key_24_iv_24 = r"""'5A75'""" + +example_mode_aes_128_ofb_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_key_24_iv_24 = r"""'76F5E35E'""" + +example_mode_aes_128_ofb_datatype_IPv6_key_24_iv_24 = r"""'7B760F4DE9B008143E1BAC7C60D22EF7'""" + +example_mode_aes_128_ofb_datatype_Enum8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_ofb_datatype_Enum16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_192_ofb_datatype_bytes_key_24_iv_16 = r"""'68'""" + +example_mode_aes_192_ofb_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_ofb_datatype_utf8string_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_192_ofb_datatype_String_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_ofb_datatype_FixedString_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_ofb_datatype_UInt8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_ofb_datatype_UInt16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_UInt32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_UInt64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Int8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Int16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_Int32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_Int64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Float32_key_24_iv_16 = r"""'68C767AD'""" + +example_mode_aes_192_ofb_datatype_Float64_key_24_iv_16 = r"""'68C7E792B710878E'""" + +example_mode_aes_192_ofb_datatype_Decimal32_key_24_iv_16 = r"""'4889E792'""" + +example_mode_aes_192_ofb_datatype_Decimal64_key_24_iv_16 = r"""'4889E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Decimal128_key_24_iv_16 = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_ofb_datatype_UUID_key_24_iv_16 = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_ofb_datatype_Date_key_24_iv_16 = r"""'3E80'""" + +example_mode_aes_192_ofb_datatype_DateTime_key_24_iv_16 = r"""'2617EBCC'""" + +example_mode_aes_192_ofb_datatype_DateTime64_key_24_iv_16 = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_ofb_datatype_Array_key_24_iv_16 = r"""'69C5'""" + +example_mode_aes_192_ofb_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_key_24_iv_16 = r"""'45450639'""" + +example_mode_aes_192_ofb_datatype_IPv6_key_24_iv_16 = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_ofb_datatype_Enum8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Enum16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_bytes_key_32_iv_32 = r"""'31'""" + +example_mode_aes_192_ofb_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_ofb_datatype_utf8string_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D86DA0963C4A14697B'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D86DA0963C4A14697B'""" + +example_mode_aes_192_ofb_datatype_String_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_ofb_datatype_FixedString_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_ofb_datatype_UInt8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_ofb_datatype_UInt16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_ofb_datatype_UInt32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_ofb_datatype_UInt64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_ofb_datatype_Int8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_ofb_datatype_Int16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_ofb_datatype_Int32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_ofb_datatype_Int64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_ofb_datatype_Float32_key_32_iv_32 = r"""'31A00E4B'""" + +example_mode_aes_192_ofb_datatype_Float64_key_32_iv_32 = r"""'31A08E74AFABC0FB'""" + +example_mode_aes_192_ofb_datatype_Decimal32_key_32_iv_32 = r"""'11EE8E74'""" + +example_mode_aes_192_ofb_datatype_Decimal64_key_32_iv_32 = r"""'11EE8E74AFAB30C4'""" + +example_mode_aes_192_ofb_datatype_Decimal128_key_32_iv_32 = r"""'11EE8E74AFAB30C4422B6D36C4A76572'""" + +example_mode_aes_192_ofb_datatype_UUID_key_32_iv_32 = r"""'D6B13D28AB6FC0A5E2F0BE5CC4011EE2'""" + +example_mode_aes_192_ofb_datatype_Date_key_32_iv_32 = r"""'67E7'""" + +example_mode_aes_192_ofb_datatype_DateTime_key_32_iv_32 = r"""'7F70822A'""" + +example_mode_aes_192_ofb_datatype_DateTime64_key_32_iv_32 = r"""'1A118316C0AA30C4'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_ofb_datatype_Array_key_32_iv_32 = r"""'30A2'""" + +example_mode_aes_192_ofb_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_key_32_iv_32 = r"""'1C226FDF'""" + +example_mode_aes_192_ofb_datatype_IPv6_key_32_iv_32 = r"""'11A183CCAFABB567422B6D3668B8E573'""" + +example_mode_aes_192_ofb_datatype_Enum8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_ofb_datatype_Enum16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_256_ofb_datatype_bytes_key_32_iv_16 = r"""'69'""" + +example_mode_aes_256_ofb_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_ofb_datatype_utf8string_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_256_ofb_datatype_String_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_ofb_datatype_FixedString_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_ofb_datatype_UInt8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_ofb_datatype_UInt16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_UInt32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_UInt64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Int8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Int16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_Int32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_Int64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Float32_key_32_iv_16 = r"""'69580828'""" + +example_mode_aes_256_ofb_datatype_Float64_key_32_iv_16 = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_ofb_datatype_Decimal32_key_32_iv_16 = r"""'49168817'""" + +example_mode_aes_256_ofb_datatype_Decimal64_key_32_iv_16 = r"""'491688173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Decimal128_key_32_iv_16 = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_ofb_datatype_UUID_key_32_iv_16 = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_ofb_datatype_Date_key_32_iv_16 = r"""'3F1F'""" + +example_mode_aes_256_ofb_datatype_DateTime_key_32_iv_16 = r"""'27888449'""" + +example_mode_aes_256_ofb_datatype_DateTime64_key_32_iv_16 = r"""'42E9857553DF4488'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_ofb_datatype_Array_key_32_iv_16 = r"""'685A'""" + +example_mode_aes_256_ofb_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_key_32_iv_16 = r"""'44DA69BC'""" + +example_mode_aes_256_ofb_datatype_IPv6_key_32_iv_16 = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_ofb_datatype_Enum8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Enum16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_bytes_key_64_iv_64 = r"""'D3'""" + +example_mode_aes_256_ofb_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_ofb_datatype_utf8string_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D063DC690F1843081'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D063DC690F1843081'""" + +example_mode_aes_256_ofb_datatype_String_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_ofb_datatype_FixedString_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_ofb_datatype_UInt8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_ofb_datatype_UInt16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_ofb_datatype_UInt32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_ofb_datatype_UInt64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_ofb_datatype_Int8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_ofb_datatype_Int16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_ofb_datatype_Int32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_ofb_datatype_Int64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_ofb_datatype_Float32_key_64_iv_64 = r"""'D3EE4F63'""" + +example_mode_aes_256_ofb_datatype_Float64_key_64_iv_64 = r"""'D3EECF5CBB79D542'""" + +example_mode_aes_256_ofb_datatype_Decimal32_key_64_iv_64 = r"""'F3A0CF5C'""" + +example_mode_aes_256_ofb_datatype_Decimal64_key_64_iv_64 = r"""'F3A0CF5CBB79257D'""" + +example_mode_aes_256_ofb_datatype_Decimal128_key_64_iv_64 = r"""'F3A0CF5CBB79257DB12F3A9075610DD7'""" + +example_mode_aes_256_ofb_datatype_UUID_key_64_iv_64 = r"""'34FF7C00BFBDD51C11F4E9FA75C77647'""" + +example_mode_aes_256_ofb_datatype_Date_key_64_iv_64 = r"""'85A9'""" + +example_mode_aes_256_ofb_datatype_DateTime_key_64_iv_64 = r"""'9D3EC302'""" + +example_mode_aes_256_ofb_datatype_DateTime64_key_64_iv_64 = r"""'F85FC23ED478257D'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_ofb_datatype_Array_key_64_iv_64 = r"""'D2EC'""" + +example_mode_aes_256_ofb_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_key_64_iv_64 = r"""'FE6C2EF7'""" + +example_mode_aes_256_ofb_datatype_IPv6_key_64_iv_64 = r"""'F3EFC2E4BB79A0DEB12F3A90D97E8DD6'""" + +example_mode_aes_256_ofb_datatype_Enum8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_ofb_datatype_Enum16_key_64_iv_64 = r"""'D2EE'""" + diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index a80ae7fc07b..6f1a5acab14 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -17,6 +17,7 @@ def regression(self, local, clickhouse_binary_path): Feature(test=load("example.regression", "regression"))(**args) Feature(test=load("ldap.regression", "regression"))(**args) Feature(test=load("rbac.regression", "regression"))(**args) + Feature(test=load("aes_encryption.regression", "regression"))(**args) if main(): regression() From 7cc4118dabb68214c0654bc66e4e202ab19adf16 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 17 Aug 2020 17:49:21 +0300 Subject: [PATCH 028/441] Fixed AES encrypt/decrypt performance Improved execution time of EVP_DecryptInit_ex/EVP_EncryptInit_ex with some hacks Fixed all-const case --- src/Functions/FunctionsAES.cpp | 8 ++++-- src/Functions/FunctionsAES.h | 40 ++++++++++++--------------- src/Functions/tests/gtest_openssl.cpp | 0 3 files changed, 23 insertions(+), 25 deletions(-) create mode 100644 src/Functions/tests/gtest_openssl.cpp diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 48533be054a..029b0727e0d 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -2,6 +2,7 @@ #if USE_SSL +#include #include #include @@ -34,7 +35,7 @@ StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const return StringRef(folded_key.data(), cipher_key_size); } -const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) +CipherPtr getCipherByName(const StringRef & cipher_name) { const auto *evp_cipher = EVP_get_cipherbyname(cipher_name.data); if (evp_cipher == nullptr) @@ -48,7 +49,10 @@ const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) evp_cipher = EVP_aes_256_cfb128(); } - return evp_cipher; + // HACK: To speed up context initialization with EVP_EncryptInit_ex (which is called at least once per row) + // Apparently cipher from EVP_get_cipherbyname may require additional initialization of context, + // while cipher from EVP_CIPHER_fetch causes less operations => faster context initialization. + return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &EVP_CIPHER_free}; } } diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 8d062e9b12d..d126bcea6d6 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -34,7 +35,9 @@ namespace OpenSSLDetails { [[noreturn]] void onError(std::string error_message); StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); -const EVP_CIPHER * getCipherByName(const StringRef & name); + +using CipherPtr = std::unique_ptr; +CipherPtr getCipherByName(const StringRef & name); enum class CompatibilityMode { @@ -148,6 +151,8 @@ private: String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -174,11 +179,6 @@ private: return std::make_shared(); } - void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override - { - block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); - } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { using namespace OpenSSLDetails; @@ -188,9 +188,10 @@ private: if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const auto * evp_cipher = getCipherByName(mode); - if (evp_cipher == nullptr) + auto cipher = getCipherByName(mode); + if (cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + const EVP_CIPHER * evp_cipher = cipher.get(); const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); @@ -371,9 +372,6 @@ private: ++encrypted; encrypted_result_column_offsets.push_back(encrypted - encrypted_result_column_data.data()); - - if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) - onError("Failed to reset context"); } // in case of block size of 1, we overestimate buffer required for encrypted data, fix it up. @@ -403,6 +401,8 @@ private: String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -429,23 +429,18 @@ private: return std::make_shared(); } - void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override - { - block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); - } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { using namespace OpenSSLDetails; const auto mode = block.getByPosition(arguments[0]).column->getDataAt(0); - if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const auto * evp_cipher = getCipherByName(mode); - if (evp_cipher == nullptr) + auto cipher = getCipherByName(mode); + if (cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + const EVP_CIPHER * evp_cipher = cipher.get(); OpenSSLDetails::validateCipherMode(evp_cipher); @@ -575,13 +570,14 @@ private: // 1: Init CTX if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) { - // 1.a.1 : Init CTX with custom IV length and optionally with AAD if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) - onError("Failed to initialize cipher context"); + onError("Failed to initialize cipher context 1"); + // 1.a.1 : Set custom IV length if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1) onError("Failed to set custom IV length to " + std::to_string(iv_value.size)); + // 1.a.1 : Init CTX with key and IV if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr, reinterpret_cast(key_value.data), reinterpret_cast(iv_value.data)) != 1) @@ -635,8 +631,6 @@ private: decrypted_result_column_offsets.push_back(decrypted - decrypted_result_column_data.data()); - if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) - onError("Failed to reset context"); } // in case we overestimate buffer required for decrypted data, fix it up. diff --git a/src/Functions/tests/gtest_openssl.cpp b/src/Functions/tests/gtest_openssl.cpp new file mode 100644 index 00000000000..e69de29bb2d From bdaa012239e73177e162c4c65ad0085c885e223c Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 31 Aug 2020 18:06:33 +0300 Subject: [PATCH 029/441] Fixed compilation for older (pre-3.0.0) OpenSSL versions --- src/Functions/FunctionsAES.cpp | 18 +++++++++++++++++- src/Functions/FunctionsAES.h | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 029b0727e0d..009226c035a 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -7,6 +7,18 @@ #include +namespace +{ +void CipherDeleter(const EVP_CIPHER * cipher [[maybe_unused]]) +{ +#if OPENSSL_VERSION_NUMBER >= 0x03 << 28 +// Used to free EVP_CIPHER poniter obtained with EVP_CIPHER_fetch, +// available only since OpenSSL ver 3.0.0. + EVP_CIPHER_free(const_cast(cipher)); +#endif +} +} + namespace DB { namespace ErrorCodes @@ -49,10 +61,14 @@ CipherPtr getCipherByName(const StringRef & cipher_name) evp_cipher = EVP_aes_256_cfb128(); } +#if OPENSSL_VERSION_NUMBER < 0x03 << 28 + return CipherPtr{evp_cipher, CipherDeleter}; +#else // HACK: To speed up context initialization with EVP_EncryptInit_ex (which is called at least once per row) // Apparently cipher from EVP_get_cipherbyname may require additional initialization of context, // while cipher from EVP_CIPHER_fetch causes less operations => faster context initialization. - return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &EVP_CIPHER_free}; + return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &CipherDeleter}; +#endif } } diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index d126bcea6d6..25b41a067a7 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -36,7 +36,8 @@ namespace OpenSSLDetails [[noreturn]] void onError(std::string error_message); StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); -using CipherPtr = std::unique_ptr; +using CipherDeleterType = void (*) (const EVP_CIPHER *cipher); +using CipherPtr = std::unique_ptr; CipherPtr getCipherByName(const StringRef & name); enum class CompatibilityMode From b6093d9a86aaf89362194be997690288d19afc8a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 1 Sep 2020 13:37:42 +0300 Subject: [PATCH 030/441] fix query header --- src/Storages/StorageMerge.cpp | 7 ++++--- src/Storages/StorageMerge.h | 15 ++++++++------- src/Storages/StorageTableFunction.h | 15 ++++++++++++++- src/TableFunctions/TableFunctionFile.cpp | 6 +++--- src/TableFunctions/TableFunctionFile.h | 4 ++-- src/TableFunctions/TableFunctionHDFS.cpp | 8 ++++---- src/TableFunctions/TableFunctionHDFS.h | 4 ++-- src/TableFunctions/TableFunctionURL.cpp | 8 ++++---- src/TableFunctions/TableFunctionURL.h | 4 ++-- 9 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 2d96a59392b..9ca93288197 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -178,7 +178,7 @@ Pipe StorageMerge::read( modified_context->setSetting("optimize_move_to_prewhere", false); /// What will be result structure depending on query processed stage in source tables? - Block header = getQueryHeader(column_names, metadata_snapshot, query_info, context, processed_stage); + Block header = getQueryHeader(*this, column_names, metadata_snapshot, query_info, context, processed_stage); /** First we make list of selected tables to find out its size. * This is necessary to correctly pass the recommended number of threads to each table. @@ -431,6 +431,7 @@ void StorageMerge::alter( } Block StorageMerge::getQueryHeader( + const IStorage & storage, const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, @@ -441,7 +442,7 @@ Block StorageMerge::getQueryHeader( { case QueryProcessingStage::FetchColumns: { - Block header = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); + Block header = metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID()); if (query_info.prewhere_info) { query_info.prewhere_info->prewhere_actions->execute(header); @@ -457,7 +458,7 @@ Block StorageMerge::getQueryHeader( removeJoin(*query->as()); auto stream = std::make_shared( - metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID())); + metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID())); return InterpreterSelectQuery(query, context, stream, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); } } diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index ad898ba2ffd..992fe309c02 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -47,6 +47,14 @@ public: bool mayBenefitFromIndexForIn( const ASTPtr & left_in_operand, const Context & query_context, const StorageMetadataPtr & metadata_snapshot) const override; + static Block getQueryHeader( + const IStorage & storage, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage); + private: String source_database; OptimizedRegularExpression table_name_regexp; @@ -75,13 +83,6 @@ protected: const String & table_name_regexp_, const Context & context_); - Block getQueryHeader( - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum processed_stage); - Pipe createSources( const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 319a2d37dca..8aa297a1e7a 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -6,6 +6,9 @@ #include #include +//#include +#include + namespace DB { @@ -25,14 +28,17 @@ public: StorageInMemoryMetadata cached_metadata; cached_metadata.setColumns(std::move(cached_columns)); setInMemoryMetadata(cached_metadata); + //log = &Poco::Logger::get("TABLE_FUNCTION_PROXY"); } StoragePtr getNested() const override { + //LOG_WARNING(log, "getNested()"); std::lock_guard lock{nested_mutex}; if (nested) return nested; + //LOG_WARNING(log, "getNested() creating"); auto nested_storage = get_nested(); nested_storage->startup(); nested = nested_storage; @@ -71,6 +77,10 @@ public: size_t max_block_size, unsigned num_streams) override { + String cnames; + for (const auto & c : column_names) + cnames += c + " "; + //LOG_WARNING(log, "read() {} cols: {}", QueryProcessingStage::toString(processed_stage), cnames); auto storage = getNested(); auto nested_metadata = storage->getInMemoryMetadataPtr(); auto pipe = storage->read(column_names, nested_metadata, query_info, context, processed_stage, max_block_size, num_streams); @@ -78,9 +88,11 @@ public: { pipe.addSimpleTransform([&](const Block & header) { + auto to = StorageMerge::getQueryHeader(*this, column_names, metadata_snapshot, query_info, context, processed_stage); + //LOG_WARNING(log, "try convert \n{}\n to \n{}\n", header.dumpStructure(), to.dumpStructure()); return std::make_shared( header, - metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()), + to, ConvertingTransform::MatchColumnsMode::Name); }); } @@ -114,6 +126,7 @@ private: mutable std::mutex nested_mutex; mutable GetNestedStorageFunc get_nested; mutable StoragePtr nested; + //mutable Poco::Logger * log; }; } diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 2814c2a940b..39de6dce92c 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -9,10 +9,10 @@ namespace DB { StoragePtr TableFunctionFile::getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const std::string & compression_method) const + const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const std::string & compression_method_) const { - StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format, compression_method, columns, ConstraintsDescription{}, global_context}; + StorageFile::CommonArguments args{StorageID(getDatabaseName(), table_name), format_, compression_method_, columns, ConstraintsDescription{}, global_context}; return StorageFile::create(source, global_context.getUserFilesPath(), args); } diff --git a/src/TableFunctions/TableFunctionFile.h b/src/TableFunctions/TableFunctionFile.h index 589b9028d96..91e2225a6d0 100644 --- a/src/TableFunctions/TableFunctionFile.h +++ b/src/TableFunctions/TableFunctionFile.h @@ -23,7 +23,7 @@ public: private: StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const std::string & compression_method) const override; + const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const std::string & compression_method_) const override; const char * getStorageTypeName() const override { return "File"; } };} diff --git a/src/TableFunctions/TableFunctionHDFS.cpp b/src/TableFunctions/TableFunctionHDFS.cpp index da4c89b1fd1..e2f227ef7b5 100644 --- a/src/TableFunctions/TableFunctionHDFS.cpp +++ b/src/TableFunctions/TableFunctionHDFS.cpp @@ -10,17 +10,17 @@ namespace DB { StoragePtr TableFunctionHDFS::getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method) const + const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method_) const { return StorageHDFS::create( source, StorageID(getDatabaseName(), table_name), - format, + format_, columns, ConstraintsDescription{}, global_context, - compression_method); + compression_method_); } diff --git a/src/TableFunctions/TableFunctionHDFS.h b/src/TableFunctions/TableFunctionHDFS.h index 7e631d1b787..47e040f7593 100644 --- a/src/TableFunctions/TableFunctionHDFS.h +++ b/src/TableFunctions/TableFunctionHDFS.h @@ -26,8 +26,8 @@ public: private: StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method) const override; + const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method_) const override; const char * getStorageTypeName() const override { return "HDFS"; } }; diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 69395c9140e..e867fc836f7 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -10,12 +10,12 @@ namespace DB { StoragePtr TableFunctionURL::getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method) const + const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method_) const { Poco::URI uri(source); - return StorageURL::create( uri, StorageID(getDatabaseName(), table_name), format, columns, ConstraintsDescription{}, - global_context, compression_method); + return StorageURL::create( uri, StorageID(getDatabaseName(), table_name), format_, columns, ConstraintsDescription{}, + global_context, compression_method_); } void registerTableFunctionURL(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index 4760e0c70f0..5eb027e2b8a 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -21,8 +21,8 @@ public: private: StoragePtr getStorage( - const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, - const std::string & table_name, const String & compression_method) const override; + const String & source, const String & format_, const ColumnsDescription & columns, Context & global_context, + const std::string & table_name, const String & compression_method_) const override; const char * getStorageTypeName() const override { return "URL"; } }; From 3a9779adb437e749433e99ac484a5033fd276163 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 1 Sep 2020 17:41:49 +0300 Subject: [PATCH 031/441] slightly better code --- src/Databases/DatabaseOnDisk.cpp | 3 +- .../ClusterProxy/SelectStreamFactory.cpp | 3 +- src/Interpreters/Context.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 9 ++- src/Interpreters/InterpreterDescribeQuery.cpp | 23 +++---- src/Interpreters/InterpreterInsertQuery.cpp | 3 +- .../getHeaderForProcessingStage.cpp | 66 +++++++++++++++++++ .../getHeaderForProcessingStage.h | 27 ++++++++ src/Storages/StorageFile.h | 2 +- src/Storages/StorageGenerateRandom.h | 2 +- src/Storages/StorageHDFS.h | 2 +- src/Storages/StorageInput.h | 2 +- src/Storages/StorageMerge.cpp | 59 +---------------- src/Storages/StorageMerge.h | 10 +-- src/Storages/StorageMySQL.h | 2 +- src/Storages/StorageTableFunction.h | 28 +++----- src/Storages/StorageURL.h | 2 +- src/Storages/StorageValues.h | 2 +- src/Storages/System/StorageSystemNumbers.h | 2 +- src/Storages/System/StorageSystemZeros.h | 2 +- src/Storages/getStructureOfRemoteTable.cpp | 6 +- src/TableFunctions/ITableFunction.cpp | 7 +- src/TableFunctions/ITableFunction.h | 7 +- src/TableFunctions/ITableFunctionFileLike.cpp | 13 ++-- src/TableFunctions/ITableFunctionFileLike.h | 12 ++-- src/TableFunctions/ITableFunctionXDBC.cpp | 16 ++--- src/TableFunctions/ITableFunctionXDBC.h | 12 ++-- src/TableFunctions/TableFunctionFactory.cpp | 13 ++-- src/TableFunctions/TableFunctionFactory.h | 2 +- .../TableFunctionGenerateRandom.cpp | 13 ++-- .../TableFunctionGenerateRandom.h | 12 ++-- src/TableFunctions/TableFunctionInput.cpp | 13 ++-- src/TableFunctions/TableFunctionInput.h | 6 +- src/TableFunctions/TableFunctionMerge.cpp | 11 ++-- src/TableFunctions/TableFunctionMerge.h | 8 +-- src/TableFunctions/TableFunctionMySQL.cpp | 12 ++-- src/TableFunctions/TableFunctionMySQL.h | 18 ++--- src/TableFunctions/TableFunctionNumbers.cpp | 2 +- src/TableFunctions/TableFunctionNumbers.h | 2 +- src/TableFunctions/TableFunctionRemote.cpp | 18 ++--- src/TableFunctions/TableFunctionRemote.h | 10 +-- src/TableFunctions/TableFunctionS3.cpp | 13 ++-- src/TableFunctions/TableFunctionS3.h | 16 ++--- src/TableFunctions/TableFunctionURL.cpp | 2 +- src/TableFunctions/TableFunctionValues.cpp | 9 +-- src/TableFunctions/TableFunctionValues.h | 6 +- src/TableFunctions/TableFunctionZeros.cpp | 2 +- src/TableFunctions/TableFunctionZeros.h | 2 +- 48 files changed, 243 insertions(+), 271 deletions(-) create mode 100644 src/Interpreters/getHeaderForProcessingStage.cpp create mode 100644 src/Interpreters/getHeaderForProcessingStage.h diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index fdfc185f920..a563f6262fc 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -54,9 +54,8 @@ std::pair createTableFromAST( if (ast_create_query.as_table_function) { - const auto & ast_table_function = ast_create_query.as_table_function->as(); const auto & factory = TableFunctionFactory::instance(); - auto table_function = factory.get(ast_table_function.name, context); + auto table_function = factory.get(ast_create_query.as_table_function, context); ColumnsDescription columns; if (ast_create_query.columns_list && ast_create_query.columns_list->columns) columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, false); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 986de85d712..9754fa7ad73 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -154,8 +154,7 @@ void SelectStreamFactory::createForShard( if (table_func_ptr) { - const auto * table_function = table_func_ptr->as(); - TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function->name, context); + TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_func_ptr, context); main_table_storage = table_function_ptr->execute(table_func_ptr, context, table_function_ptr->getName()); } else diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 70cf41a679c..2cd65a62d25 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -916,7 +916,7 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression) if (!res) { - TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_expression->as()->name, *this); + TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_expression, *this); /// Run it and remember the result res = table_function_ptr->execute(table_expression, *this, table_function_ptr->getName()); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index dd5fb4d063a..c6ad6c5bb2b 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -497,9 +497,9 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS else if (create.as_table_function) { /// Table function without columns list. - auto table_function = TableFunctionFactory::instance().get(create.as_table_function->as().name, context); - properties.columns = table_function->getActualTableStructure(create.as_table_function, context); - if (properties.columns.empty()) //FIXME + auto table_function = TableFunctionFactory::instance().get(create.as_table_function, context); + properties.columns = table_function->getActualTableStructure(context); + if (properties.columns.empty()) //FIXME TableFunctionFile may return empty structure for Distributed format return {}; } else @@ -768,9 +768,8 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) { - const auto & table_function = create.as_table_function->as(); const auto & factory = TableFunctionFactory::instance(); - res = factory.get(table_function.name, context)->execute(create.as_table_function, context, create.table, properties.columns); + res = factory.get(create.as_table_function, context)->execute(create.as_table_function, context, create.table, properties.columns); res->renameInMemory({create.database, create.table, create.uuid}); } else diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp index 94fa748ea15..b540290a653 100644 --- a/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/src/Interpreters/InterpreterDescribeQuery.cpp @@ -72,23 +72,16 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() table_expression.subquery->children.at(0), context).getNamesAndTypesList(); columns = ColumnsDescription(std::move(names_and_types)); } + else if (table_expression.table_function) + { + TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_expression.table_function, context); + columns = table_function_ptr->getActualTableStructure(context); + } else { - StoragePtr table; - if (table_expression.table_function) - { - const auto & table_function = table_expression.table_function->as(); - TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function.name, context); - /// Run the table function and remember the result - table = table_function_ptr->execute(table_expression.table_function, context, table_function_ptr->getName()); - } - else - { - auto table_id = context.resolveStorageID(table_expression.database_and_table_name); - context.checkAccess(AccessType::SHOW_COLUMNS, table_id); - table = DatabaseCatalog::instance().getTable(table_id, context); - } - + auto table_id = context.resolveStorageID(table_expression.database_and_table_name); + context.checkAccess(AccessType::SHOW_COLUMNS, table_id); + auto table = DatabaseCatalog::instance().getTable(table_id, context); auto table_lock = table->lockForShare(context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout); auto metadata_snapshot = table->getInMemoryMetadataPtr(); columns = metadata_snapshot->getColumns(); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 1557f065b32..b1ee7cfe0b1 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -59,9 +59,8 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) { if (query.table_function) { - const auto * table_function = query.table_function->as(); const auto & factory = TableFunctionFactory::instance(); - TableFunctionPtr table_function_ptr = factory.get(table_function->name, context); + TableFunctionPtr table_function_ptr = factory.get(query.table_function, context); return table_function_ptr->execute(query.table_function, context, table_function_ptr->getName()); } diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp new file mode 100644 index 00000000000..d034f900842 --- /dev/null +++ b/src/Interpreters/getHeaderForProcessingStage.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +/// Rewrite original query removing joined tables from it +void removeJoin(const ASTSelectQuery & select) +{ + const auto & tables = select.tables(); + if (!tables || tables->children.size() < 2) + return; + + const auto & joined_table = tables->children[1]->as(); + if (!joined_table.table_join) + return; + + /// The most simple temporary solution: leave only the first table in query. + /// TODO: we also need to remove joined columns and related functions (taking in account aliases if any). + tables->children.resize(1); +} + +Block getHeaderForProcessingStage( + const IStorage & storage, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage) +{ + switch (processed_stage) + { + case QueryProcessingStage::FetchColumns: + { + Block header = metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID()); + if (query_info.prewhere_info) + { + query_info.prewhere_info->prewhere_actions->execute(header); + if (query_info.prewhere_info->remove_prewhere_column) + header.erase(query_info.prewhere_info->prewhere_column_name); + } + return header; + } + case QueryProcessingStage::WithMergeableState: + case QueryProcessingStage::Complete: + { + auto query = query_info.query->clone(); + removeJoin(*query->as()); + + auto stream = std::make_shared( + metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID())); + return InterpreterSelectQuery(query, context, stream, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); + } + } + throw Exception("Logical Error: unknown processed stage.", ErrorCodes::LOGICAL_ERROR); +} + +} + diff --git a/src/Interpreters/getHeaderForProcessingStage.h b/src/Interpreters/getHeaderForProcessingStage.h new file mode 100644 index 00000000000..a4ebcc1db7d --- /dev/null +++ b/src/Interpreters/getHeaderForProcessingStage.h @@ -0,0 +1,27 @@ +#pragma once +#include +#include +#include + + +namespace DB +{ + +class IStorage; +struct StorageInMemoryMetadata; +using StorageMetadataPtr = std::shared_ptr; +struct SelectQueryInfo; +class Context; +class ASTSelectQuery; + +void removeJoin(const ASTSelectQuery & select); + +Block getHeaderForProcessingStage( + const IStorage & storage, + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage); + +} diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 872a0fc163f..ea70dcd5311 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -18,7 +18,7 @@ namespace DB class StorageFileBlockInputStream; class StorageFileBlockOutputStream; -class StorageFile : public ext::shared_ptr_helper, public IStorage +class StorageFile final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageGenerateRandom.h b/src/Storages/StorageGenerateRandom.h index 09208e9c5cd..e0f037f9a08 100644 --- a/src/Storages/StorageGenerateRandom.h +++ b/src/Storages/StorageGenerateRandom.h @@ -9,7 +9,7 @@ namespace DB { /* Generates random data for given schema. */ -class StorageGenerateRandom : public ext::shared_ptr_helper, public IStorage +class StorageGenerateRandom final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageHDFS.h b/src/Storages/StorageHDFS.h index ecb77ef83af..fdeaf4ae1b3 100644 --- a/src/Storages/StorageHDFS.h +++ b/src/Storages/StorageHDFS.h @@ -13,7 +13,7 @@ namespace DB * This class represents table engine for external hdfs files. * Read method is supported for now. */ -class StorageHDFS : public ext::shared_ptr_helper, public IStorage +class StorageHDFS final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageInput.h b/src/Storages/StorageInput.h index 40b0fd39139..c19b19e4703 100644 --- a/src/Storages/StorageInput.h +++ b/src/Storages/StorageInput.h @@ -8,7 +8,7 @@ namespace DB /** Internal temporary storage for table function input(...) */ -class StorageInput : public ext::shared_ptr_helper, public IStorage +class StorageInput final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 9ca93288197..9caec05e2f4 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -37,27 +38,6 @@ namespace ErrorCodes extern const int SAMPLING_NOT_SUPPORTED; } -namespace -{ - -/// Rewrite original query removing joined tables from it -void removeJoin(const ASTSelectQuery & select) -{ - const auto & tables = select.tables(); - if (!tables || tables->children.size() < 2) - return; - - const auto & joined_table = tables->children[1]->as(); - if (!joined_table.table_join) - return; - - /// The most simple temporary solution: leave only the first table in query. - /// TODO: we also need to remove joined columns and related functions (taking in account aliases if any). - tables->children.resize(1); -} - -} - StorageMerge::StorageMerge( const StorageID & table_id_, @@ -178,7 +158,7 @@ Pipe StorageMerge::read( modified_context->setSetting("optimize_move_to_prewhere", false); /// What will be result structure depending on query processed stage in source tables? - Block header = getQueryHeader(*this, column_names, metadata_snapshot, query_info, context, processed_stage); + Block header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, query_info, context, processed_stage); /** First we make list of selected tables to find out its size. * This is necessary to correctly pass the recommended number of threads to each table. @@ -430,41 +410,6 @@ void StorageMerge::alter( setInMemoryMetadata(storage_metadata); } -Block StorageMerge::getQueryHeader( - const IStorage & storage, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum processed_stage) -{ - switch (processed_stage) - { - case QueryProcessingStage::FetchColumns: - { - Block header = metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID()); - if (query_info.prewhere_info) - { - query_info.prewhere_info->prewhere_actions->execute(header); - if (query_info.prewhere_info->remove_prewhere_column) - header.erase(query_info.prewhere_info->prewhere_column_name); - } - return header; - } - case QueryProcessingStage::WithMergeableState: - case QueryProcessingStage::Complete: - { - auto query = query_info.query->clone(); - removeJoin(*query->as()); - - auto stream = std::make_shared( - metadata_snapshot->getSampleBlockForColumns(column_names, storage.getVirtuals(), storage.getStorageID())); - return InterpreterSelectQuery(query, context, stream, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); - } - } - throw Exception("Logical Error: unknown processed stage.", ErrorCodes::LOGICAL_ERROR); -} - void StorageMerge::convertingSourceStream( const Block & header, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 992fe309c02..63d803cedf1 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -13,7 +13,7 @@ namespace DB /** A table that represents the union of an arbitrary number of other tables. * All tables must have the same structure. */ -class StorageMerge : public ext::shared_ptr_helper, public IStorage +class StorageMerge final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: @@ -47,14 +47,6 @@ public: bool mayBenefitFromIndexForIn( const ASTPtr & left_in_operand, const Context & query_context, const StorageMetadataPtr & metadata_snapshot) const override; - static Block getQueryHeader( - const IStorage & storage, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum processed_stage); - private: String source_database; OptimizedRegularExpression table_name_regexp; diff --git a/src/Storages/StorageMySQL.h b/src/Storages/StorageMySQL.h index fedd8467210..a7f98c4379b 100644 --- a/src/Storages/StorageMySQL.h +++ b/src/Storages/StorageMySQL.h @@ -20,7 +20,7 @@ namespace DB * Use ENGINE = mysql(host_port, database_name, table_name, user_name, password) * Read only. */ -class StorageMySQL : public ext::shared_ptr_helper, public IStorage +class StorageMySQL final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 8aa297a1e7a..530e614c856 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -5,9 +5,8 @@ #include #include #include +#include -//#include -#include namespace DB { @@ -28,17 +27,14 @@ public: StorageInMemoryMetadata cached_metadata; cached_metadata.setColumns(std::move(cached_columns)); setInMemoryMetadata(cached_metadata); - //log = &Poco::Logger::get("TABLE_FUNCTION_PROXY"); } StoragePtr getNested() const override { - //LOG_WARNING(log, "getNested()"); std::lock_guard lock{nested_mutex}; if (nested) return nested; - //LOG_WARNING(log, "getNested() creating"); auto nested_storage = get_nested(); nested_storage->startup(); nested = nested_storage; @@ -46,12 +42,6 @@ public: return nested; } - StoragePtr maybeGetNested() const - { - std::lock_guard lock{nested_mutex}; - return nested; - } - String getName() const override { std::lock_guard lock{nested_mutex}; @@ -63,9 +53,9 @@ public: void startup() override { } void shutdown() override { - auto storage = maybeGetNested(); - if (storage) - storage->shutdown(); + std::lock_guard lock{nested_mutex}; + if (nested) + nested->shutdown(); } Pipe read( @@ -80,19 +70,17 @@ public: String cnames; for (const auto & c : column_names) cnames += c + " "; - //LOG_WARNING(log, "read() {} cols: {}", QueryProcessingStage::toString(processed_stage), cnames); auto storage = getNested(); auto nested_metadata = storage->getInMemoryMetadataPtr(); auto pipe = storage->read(column_names, nested_metadata, query_info, context, processed_stage, max_block_size, num_streams); if (!pipe.empty()) { + auto to_header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, query_info, context, processed_stage); pipe.addSimpleTransform([&](const Block & header) { - auto to = StorageMerge::getQueryHeader(*this, column_names, metadata_snapshot, query_info, context, processed_stage); - //LOG_WARNING(log, "try convert \n{}\n to \n{}\n", header.dumpStructure(), to.dumpStructure()); return std::make_shared( header, - to, + to_header, ConvertingTransform::MatchColumnsMode::Name); }); } @@ -116,7 +104,8 @@ public: void renameInMemory(const StorageID & new_table_id) override { - if (maybeGetNested()) + std::lock_guard lock{nested_mutex}; + if (nested) StorageProxy::renameInMemory(new_table_id); else IStorage::renameInMemory(new_table_id); @@ -126,7 +115,6 @@ private: mutable std::mutex nested_mutex; mutable GetNestedStorageFunc get_nested; mutable StoragePtr nested; - //mutable Poco::Logger * log; }; } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 69fe19650eb..7983ad71520 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -93,7 +93,7 @@ private: BlockOutputStreamPtr writer; }; -class StorageURL : public ext::shared_ptr_helper, public IStorageURLBase +class StorageURL final : public ext::shared_ptr_helper, public IStorageURLBase { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/StorageValues.h b/src/Storages/StorageValues.h index b69af140567..8a1a06eeb54 100644 --- a/src/Storages/StorageValues.h +++ b/src/Storages/StorageValues.h @@ -9,7 +9,7 @@ namespace DB /* One block storage used for values table function * It's structure is similar to IStorageSystemOneBlock */ -class StorageValues : public ext::shared_ptr_helper, public IStorage +class StorageValues final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/System/StorageSystemNumbers.h b/src/Storages/System/StorageSystemNumbers.h index cdffd6c8434..f907f3d5f93 100644 --- a/src/Storages/System/StorageSystemNumbers.h +++ b/src/Storages/System/StorageSystemNumbers.h @@ -23,7 +23,7 @@ class Context; * In multithreaded case, if even_distributed is False, implementation with atomic is used, * and result is always in [0 ... limit - 1] range. */ -class StorageSystemNumbers : public ext::shared_ptr_helper, public IStorage +class StorageSystemNumbers final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/System/StorageSystemZeros.h b/src/Storages/System/StorageSystemZeros.h index 26379df5956..41de3ce6246 100644 --- a/src/Storages/System/StorageSystemZeros.h +++ b/src/Storages/System/StorageSystemZeros.h @@ -14,7 +14,7 @@ namespace DB * You could also specify a limit (how many zeros to give). * If multithreaded is specified, zeros will be generated in several streams. */ -class StorageSystemZeros : public ext::shared_ptr_helper, public IStorage +class StorageSystemZeros final : public ext::shared_ptr_helper, public IStorage { friend struct ext::shared_ptr_helper; public: diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp index 7dfee90588a..ea1be8223b2 100644 --- a/src/Storages/getStructureOfRemoteTable.cpp +++ b/src/Storages/getStructureOfRemoteTable.cpp @@ -73,10 +73,8 @@ ColumnsDescription getStructureOfRemoteTableInShard( { if (shard_info.isLocal()) { - const auto * table_function = table_func_ptr->as(); - TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_function->name, context); - auto storage_ptr = table_function_ptr->execute(table_func_ptr, context, table_function_ptr->getName()); - return storage_ptr->getInMemoryMetadataPtr()->getColumns(); + TableFunctionPtr table_function_ptr = TableFunctionFactory::instance().get(table_func_ptr, context); + return table_function_ptr->getActualTableStructure(context); } auto table_func_name = queryToString(table_func_ptr); diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index d5a78bda08a..13282275322 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -14,13 +14,12 @@ namespace ProfileEvents namespace DB { -StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns_) const +StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const { ProfileEvents::increment(ProfileEvents::TableFunctionExecute); context.checkAccess(AccessType::CREATE_TEMPORARY_TABLE | StorageFactory::instance().getSourceAccessType(getStorageTypeName())); - cached_columns = std::move(cached_columns_); - bool no_conversion_required = hasStaticStructure() && cached_columns == getActualTableStructure(ast_function, context); + bool no_conversion_required = hasStaticStructure() && cached_columns == getActualTableStructure(context); if (cached_columns.empty() || no_conversion_required) return executeImpl(ast_function, context, table_name); @@ -29,7 +28,7 @@ StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & return tf->executeImpl(ast_function, context, table_name); }; - return std::make_shared(StorageID(getDatabaseName(), table_name), std::move(get_storage), cached_columns); + return std::make_shared(StorageID(getDatabaseName(), table_name), std::move(get_storage), std::move(cached_columns)); } } diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index be1593fcaff..b41213a5e09 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -34,7 +34,9 @@ public: virtual bool hasStaticStructure() const { return false; } - virtual ColumnsDescription getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const = 0; + virtual void parseArguments(const ASTPtr & /*ast_function*/, const Context & /*context*/) {} + + virtual ColumnsDescription getActualTableStructure(const Context & /*context*/) const = 0; /// Create storage according to the query. StoragePtr execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns_ = {}) const; @@ -44,9 +46,6 @@ public: private: virtual StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const = 0; virtual const char * getStorageTypeName() const = 0; - -protected: - mutable ColumnsDescription cached_columns; }; using TableFunctionPtr = std::shared_ptr; diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 22c537bca2f..b362a646f9e 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -22,11 +22,8 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const Context & context) const +void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const Context & context) { - if (!filename.empty()) - return; - /// Parse args ASTs & args_func = ast_function->children; @@ -60,18 +57,16 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const C compression_method = args[3]->as().value.safeGet(); } -StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - auto columns = getActualTableStructure(ast_function, context); + auto columns = getActualTableStructure(context); StoragePtr storage = getStorage(filename, format, columns, const_cast(context), table_name, compression_method); storage->startup(); return storage; } -ColumnsDescription ITableFunctionFileLike::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription ITableFunctionFileLike::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); if (structure.empty()) { assert(getName() == "file" && format == "Distributed"); diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 47a41c8d1bb..40111eebc5c 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -20,13 +20,13 @@ private: const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, const std::string & table_name, const String & compression_method) const = 0; - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + ColumnsDescription getActualTableStructure(const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String filename; - mutable String format; - mutable String structure; - mutable String compression_method = "auto"; + String filename; + String format; + String structure; + String compression_method = "auto"; }; } diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index 6f1c1f5b2db..6ba62b8ebf1 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -25,15 +25,11 @@ namespace DB namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int UNKNOWN_EXCEPTION; extern const int LOGICAL_ERROR; } -void ITableFunctionXDBC::parseArguments(const ASTPtr & ast_function, const Context & context) const +void ITableFunctionXDBC::parseArguments(const ASTPtr & ast_function, const Context & context) { - if (helper) - return; - const auto & args_func = ast_function->as(); if (!args_func.arguments) @@ -65,9 +61,9 @@ void ITableFunctionXDBC::parseArguments(const ASTPtr & ast_function, const Conte helper->startBridgeSync(); } -ColumnsDescription ITableFunctionXDBC::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription ITableFunctionXDBC::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); + assert(helper); /* Infer external table structure */ Poco::URI columns_info_uri = helper->getColumnsInfoURI(); @@ -89,10 +85,10 @@ ColumnsDescription ITableFunctionXDBC::getActualTableStructure(const ASTPtr & as return ColumnsDescription{columns}; } -StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - auto columns = getActualTableStructure(ast_function, context); + assert(helper); + auto columns = getActualTableStructure(context); auto result = std::make_shared(StorageID(getDatabaseName(), table_name), schema_name, remote_table_name, columns, context, helper); result->startup(); return result; diff --git a/src/TableFunctions/ITableFunctionXDBC.h b/src/TableFunctions/ITableFunctionXDBC.h index f2b3e70a3a9..218f361c025 100644 --- a/src/TableFunctions/ITableFunctionXDBC.h +++ b/src/TableFunctions/ITableFunctionXDBC.h @@ -25,14 +25,14 @@ private: const Poco::Timespan & http_timeout_, const std::string & connection_string_) const = 0; - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + ColumnsDescription getActualTableStructure(const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String connection_string; - mutable String schema_name; - mutable String remote_table_name; - mutable BridgeHelperPtr helper; + String connection_string; + String schema_name; + String remote_table_name; + BridgeHelperPtr helper; }; class TableFunctionJDBC : public ITableFunctionXDBC diff --git a/src/TableFunctions/TableFunctionFactory.cpp b/src/TableFunctions/TableFunctionFactory.cpp index bc139edfb73..0399d83fe4d 100644 --- a/src/TableFunctions/TableFunctionFactory.cpp +++ b/src/TableFunctions/TableFunctionFactory.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -28,19 +29,21 @@ void TableFunctionFactory::registerFunction(const std::string & name, Value crea } TableFunctionPtr TableFunctionFactory::get( - const std::string & name, + const ASTPtr & ast_function, const Context & context) const { - auto res = tryGet(name, context); + const auto * table_function = ast_function->as(); + auto res = tryGet(table_function->name, context); if (!res) { - auto hints = getHints(name); + auto hints = getHints(table_function->name); if (!hints.empty()) - throw Exception("Unknown table function " + name + ". Maybe you meant: " + toString(hints), ErrorCodes::UNKNOWN_FUNCTION); + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unknown table function {}. Maybe you meant: {}", table_function->name , toString(hints)); else - throw Exception("Unknown table function " + name, ErrorCodes::UNKNOWN_FUNCTION); + throw Exception(ErrorCodes::UNKNOWN_FUNCTION, "Unknown table function {}", table_function->name); } + res->parseArguments(ast_function, context); return res; } diff --git a/src/TableFunctions/TableFunctionFactory.h b/src/TableFunctions/TableFunctionFactory.h index 6d0302a64ff..820b5eb1c7b 100644 --- a/src/TableFunctions/TableFunctionFactory.h +++ b/src/TableFunctions/TableFunctionFactory.h @@ -41,7 +41,7 @@ public: } /// Throws an exception if not found. - TableFunctionPtr get(const std::string & name, const Context & context) const; + TableFunctionPtr get(const ASTPtr & ast_function, const Context & context) const; /// Returns nullptr if not found. TableFunctionPtr tryGet(const std::string & name, const Context & context) const; diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index eb741ad0ba1..c88cb3229b5 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -29,11 +29,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) const +void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) { - if (!structure.empty()) - return; - ASTs & args_func = ast_function->children; if (args_func.size() != 1) @@ -80,16 +77,14 @@ void TableFunctionGenerateRandom::parseArguments(const ASTPtr & ast_function, co max_array_length = args[3]->as().value.safeGet(); } -ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - auto columns = getActualTableStructure(ast_function, context); + auto columns = getActualTableStructure(context); auto res = StorageGenerateRandom::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); res->startup(); return res; diff --git a/src/TableFunctions/TableFunctionGenerateRandom.h b/src/TableFunctions/TableFunctionGenerateRandom.h index 45ad93ffee7..8c13f853a0d 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.h +++ b/src/TableFunctions/TableFunctionGenerateRandom.h @@ -17,13 +17,13 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "GenerateRandom"; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + ColumnsDescription getActualTableStructure(const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String structure; - mutable UInt64 max_string_length = 10; - mutable UInt64 max_array_length = 10; - mutable std::optional random_seed; + String structure; + UInt64 max_string_length = 10; + UInt64 max_array_length = 10; + std::optional random_seed; }; diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index 4d0f8df55e6..6563d8f0b3a 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -24,11 +24,8 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -void TableFunctionInput::parseArguments(const ASTPtr & ast_function, const Context & context) const +void TableFunctionInput::parseArguments(const ASTPtr & ast_function, const Context & context) { - if (!structure.empty()) - return; - const auto * function = ast_function->as(); if (!function->arguments) @@ -43,16 +40,14 @@ void TableFunctionInput::parseArguments(const ASTPtr & ast_function, const Conte structure = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context)->as().value.safeGet(); } -ColumnsDescription TableFunctionInput::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionInput::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionInput::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionInput::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - auto storage = StorageInput::create(StorageID(getDatabaseName(), table_name), getActualTableStructure(ast_function, context)); + auto storage = StorageInput::create(StorageID(getDatabaseName(), table_name), getActualTableStructure(context)); storage->startup(); return storage; } diff --git a/src/TableFunctions/TableFunctionInput.h b/src/TableFunctions/TableFunctionInput.h index 50744d3ca14..9190fe600f6 100644 --- a/src/TableFunctions/TableFunctionInput.h +++ b/src/TableFunctions/TableFunctionInput.h @@ -20,10 +20,10 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Input"; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + ColumnsDescription getActualTableStructure(const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String structure; + String structure; }; } diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 360ec583740..6358a0bbbef 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -46,7 +46,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str return any_table->getInMemoryMetadataPtr()->getColumns().getAllPhysical(); } -void TableFunctionMerge::parseArguments(const ASTPtr & ast_function, const Context & context) const +void TableFunctionMerge::parseArguments(const ASTPtr & ast_function, const Context & context) { ASTs & args_func = ast_function->children; @@ -69,19 +69,16 @@ void TableFunctionMerge::parseArguments(const ASTPtr & ast_function, const Conte table_name_regexp = args[1]->as().value.safeGet(); } -ColumnsDescription TableFunctionMerge::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionMerge::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); return ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}; } -StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - auto res = StorageMerge::create( StorageID(getDatabaseName(), table_name), - getActualTableStructure(ast_function, context), + getActualTableStructure(context), source_database, table_name_regexp, context); diff --git a/src/TableFunctions/TableFunctionMerge.h b/src/TableFunctions/TableFunctionMerge.h index 0e6878983e3..2d038de32fb 100644 --- a/src/TableFunctions/TableFunctionMerge.h +++ b/src/TableFunctions/TableFunctionMerge.h @@ -19,11 +19,11 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Merge"; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + ColumnsDescription getActualTableStructure(const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String source_database; - mutable String table_name_regexp; + String source_database; + String table_name_regexp; }; diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index bff2842c8c1..c957bb8c05f 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -34,7 +34,7 @@ namespace ErrorCodes extern const int UNKNOWN_TABLE; } -void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, const Context & context) const +void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, const Context & context) { const auto & args_func = ast_function->as(); @@ -70,9 +70,9 @@ void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, const Conte parsed_host_port = parseAddress(host_port, 3306); } -ColumnsDescription TableFunctionMySQL::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionMySQL::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); + assert(!parsed_host_port.first.empty()); if (!pool) pool.emplace(remote_database_name, parsed_host_port.first, user_name, password, parsed_host_port.second); @@ -121,13 +121,13 @@ ColumnsDescription TableFunctionMySQL::getActualTableStructure(const ASTPtr & as return ColumnsDescription{columns}; } -StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); + assert(!parsed_host_port.first.empty()); if (!pool) pool.emplace(remote_database_name, parsed_host_port.first, user_name, password, parsed_host_port.second); - auto columns = getActualTableStructure(ast_function, context); + auto columns = getActualTableStructure(context); auto res = StorageMySQL::create( StorageID(getDatabaseName(), table_name), diff --git a/src/TableFunctions/TableFunctionMySQL.h b/src/TableFunctions/TableFunctionMySQL.h index e8f5851b502..f3c15a5fdd9 100644 --- a/src/TableFunctions/TableFunctionMySQL.h +++ b/src/TableFunctions/TableFunctionMySQL.h @@ -27,16 +27,16 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "MySQL"; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + ColumnsDescription getActualTableStructure(const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable std::pair parsed_host_port; - mutable String remote_database_name; - mutable String remote_table_name; - mutable String user_name; - mutable String password; - mutable bool replace_query = false; - mutable String on_duplicate_clause; + std::pair parsed_host_port; + String remote_database_name; + String remote_table_name; + String user_name; + String password; + bool replace_query = false; + String on_duplicate_clause; mutable std::optional pool; }; diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 03756b37b07..84edd3935b3 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -22,7 +22,7 @@ namespace ErrorCodes template -ColumnsDescription TableFunctionNumbers::getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const +ColumnsDescription TableFunctionNumbers::getActualTableStructure(const Context & /*context*/) const { return ColumnsDescription({{"number", std::make_shared()}}); } diff --git a/src/TableFunctions/TableFunctionNumbers.h b/src/TableFunctions/TableFunctionNumbers.h index a3c4f6f13cb..5cbde6cf0a3 100644 --- a/src/TableFunctions/TableFunctionNumbers.h +++ b/src/TableFunctions/TableFunctionNumbers.h @@ -23,7 +23,7 @@ private: UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + ColumnsDescription getActualTableStructure(const Context & context) const override; }; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 3d00d0ad897..594b47a3666 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -28,11 +28,8 @@ namespace ErrorCodes } -void TableFunctionRemote::prepareClusterInfo(const ASTPtr & ast_function, const Context & context) const +void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, const Context & context) { - if (cluster) - return; - ASTs & args_func = ast_function->children; if (args_func.size() != 1) @@ -198,14 +195,13 @@ void TableFunctionRemote::prepareClusterInfo(const ASTPtr & ast_function, const remote_table_id.table_name = remote_table; } -StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - prepareClusterInfo(ast_function, context); - + assert(cluster); StoragePtr res = remote_table_function_ptr ? StorageDistributed::create( StorageID(getDatabaseName(), table_name), - getActualTableStructure(ast_function, context), + getActualTableStructure(context), ConstraintsDescription{}, remote_table_function_ptr, String{}, @@ -217,7 +213,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C cluster) : StorageDistributed::create( StorageID(getDatabaseName(), table_name), - getActualTableStructure(ast_function, context), + getActualTableStructure(context), ConstraintsDescription{}, remote_table_id.database_name, remote_table_id.table_name, @@ -233,9 +229,9 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C return res; } -ColumnsDescription TableFunctionRemote::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionRemote::getActualTableStructure(const Context & context) const { - prepareClusterInfo(ast_function, context); + assert(cluster); return getStructureOfRemoteTable(*cluster, remote_table_id, context, remote_table_function_ptr); } diff --git a/src/TableFunctions/TableFunctionRemote.h b/src/TableFunctions/TableFunctionRemote.h index 35d1ca3b9ff..ff214bb7332 100644 --- a/src/TableFunctions/TableFunctionRemote.h +++ b/src/TableFunctions/TableFunctionRemote.h @@ -22,22 +22,22 @@ public: std::string getName() const override { return name; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + ColumnsDescription getActualTableStructure(const Context & context) const override; private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Distributed"; } - void prepareClusterInfo(const ASTPtr & ast_function, const Context & context) const; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; std::string name; bool is_cluster_function; std::string help_message; bool secure; - mutable ClusterPtr cluster; - mutable StorageID remote_table_id = StorageID::createEmpty(); - mutable ASTPtr remote_table_function_ptr; + ClusterPtr cluster; + StorageID remote_table_id = StorageID::createEmpty(); + ASTPtr remote_table_function_ptr; }; } diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index b4fe1baffce..a4f403f7875 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -22,10 +22,8 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } -void TableFunctionS3::parseArguments(const ASTPtr & ast_function, const Context & context) const +void TableFunctionS3::parseArguments(const ASTPtr & ast_function, const Context & context) { - - /// Parse args ASTs & args_func = ast_function->children; @@ -60,16 +58,13 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, const Context compression_method = args.back()->as().value.safeGet(); } -ColumnsDescription TableFunctionS3::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionS3::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - Poco::URI uri (filename); S3::URI s3_uri (uri); UInt64 min_upload_part_size = context.getSettingsRef().s3_min_upload_part_size; @@ -81,7 +76,7 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & ast_function, const Conte StorageID(getDatabaseName(), table_name), format, min_upload_part_size, - getActualTableStructure(ast_function, context), + getActualTableStructure(context), ConstraintsDescription{}, const_cast(context), compression_method); diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 23dec9a689c..45738de8ba8 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -31,15 +31,15 @@ protected: const char * getStorageTypeName() const override { return "S3"; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + ColumnsDescription getActualTableStructure(const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String filename; - mutable String format; - mutable String structure; - mutable String access_key_id; - mutable String secret_access_key; - mutable String compression_method = "auto"; + String filename; + String format; + String structure; + String access_key_id; + String secret_access_key; + String compression_method = "auto"; }; class TableFunctionCOS : public TableFunctionS3 diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index e867fc836f7..6139e6ffecb 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -14,7 +14,7 @@ StoragePtr TableFunctionURL::getStorage( const std::string & table_name, const String & compression_method_) const { Poco::URI uri(source); - return StorageURL::create( uri, StorageID(getDatabaseName(), table_name), format_, columns, ConstraintsDescription{}, + return StorageURL::create(uri, StorageID(getDatabaseName(), table_name), format_, columns, ConstraintsDescription{}, global_context, compression_method_); } diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index e51e28abff3..4e892c83247 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -63,7 +63,7 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args } } -void TableFunctionValues::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) const +void TableFunctionValues::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) { @@ -90,17 +90,14 @@ void TableFunctionValues::parseArguments(const ASTPtr & ast_function, const Cont structure = args[0]->as().value.safeGet(); } -ColumnsDescription TableFunctionValues::getActualTableStructure(const ASTPtr & ast_function, const Context & context) const +ColumnsDescription TableFunctionValues::getActualTableStructure(const Context & context) const { - parseArguments(ast_function, context); return parseColumnsListFromString(structure, context); } StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const { - parseArguments(ast_function, context); - - auto columns = getActualTableStructure(ast_function, context); + auto columns = getActualTableStructure(context); Block sample_block; for (const auto & name_type : columns.getOrdinary()) diff --git a/src/TableFunctions/TableFunctionValues.h b/src/TableFunctions/TableFunctionValues.h index 9ab2dfb047e..b66f5bb457f 100644 --- a/src/TableFunctions/TableFunctionValues.h +++ b/src/TableFunctions/TableFunctionValues.h @@ -16,10 +16,10 @@ private: StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; const char * getStorageTypeName() const override { return "Values"; } - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; - void parseArguments(const ASTPtr & ast_function, const Context & context) const; + ColumnsDescription getActualTableStructure(const Context & context) const override; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; - mutable String structure; + String structure; }; diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp index b633260861c..5ba6c034e09 100644 --- a/src/TableFunctions/TableFunctionZeros.cpp +++ b/src/TableFunctions/TableFunctionZeros.cpp @@ -21,7 +21,7 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; template -ColumnsDescription TableFunctionZeros::getActualTableStructure(const ASTPtr & /*ast_function*/, const Context & /*context*/) const +ColumnsDescription TableFunctionZeros::getActualTableStructure(const Context & /*context*/) const { return ColumnsDescription({{"zero", std::make_shared()}}); } diff --git a/src/TableFunctions/TableFunctionZeros.h b/src/TableFunctions/TableFunctionZeros.h index aba161af4d0..c50a17d046d 100644 --- a/src/TableFunctions/TableFunctionZeros.h +++ b/src/TableFunctions/TableFunctionZeros.h @@ -23,7 +23,7 @@ private: UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; - ColumnsDescription getActualTableStructure(const ASTPtr & ast_function, const Context & context) const override; + ColumnsDescription getActualTableStructure(const Context & context) const override; }; From c7f355a8f3701ae6d090e274ebd07d945af3362c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 1 Sep 2020 18:24:21 +0300 Subject: [PATCH 032/441] add test --- src/Storages/StorageTableFunction.h | 10 +++++++ ..._expressions_in_engine_arguments.reference | 6 ++-- tests/queries/0_stateless/014 | 0 ...eate_as_table_function_structure.reference | 7 +++++ ...457_create_as_table_function_structure.sql | 28 +++++++++++++++++++ 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/014 create mode 100644 tests/queries/0_stateless/01457_create_as_table_function_structure.reference create mode 100644 tests/queries/0_stateless/01457_create_as_table_function_structure.sql diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 530e614c856..87af3a49a5c 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -58,6 +58,13 @@ public: nested->shutdown(); } + void drop() override + { + std::lock_guard lock{nested_mutex}; + if (nested) + nested->drop(); + } + Pipe read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -111,6 +118,9 @@ public: IStorage::renameInMemory(new_table_id); } + bool isView() const override { return false; } + void checkTableCanBeDropped() const override {} + private: mutable std::mutex nested_mutex; mutable GetNestedStorageFunc get_nested; diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference index 933fda604d7..2a5d7e6da32 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.reference @@ -1,11 +1,11 @@ CREATE TABLE default.file\n(\n `n` Int8\n)\nENGINE = File(\'TSVWithNamesAndTypes\') CREATE TABLE default.buffer\n(\n `n` Int8\n)\nENGINE = Buffer(\'default\', \'file\', 16, 10, 200, 10000, 1000000, 10000000, 1000000000) CREATE TABLE default.merge\n(\n `n` Int8\n)\nENGINE = Merge(\'default\', \'distributed\') -CREATE TABLE default.merge_tf AS merge(\'default\', \'.*\') +CREATE TABLE default.merge_tf\n(\n `n` Int8\n) AS merge(\'default\', \'.*\') CREATE TABLE default.distributed\n(\n `n` Int8\n)\nENGINE = Distributed(\'test_shard_localhost\', \'default\', \'file\') -CREATE TABLE default.distributed_tf AS cluster(\'test_shard_localhost\', \'default\', \'buffer\') +CREATE TABLE default.distributed_tf\n(\n `n` Int8\n) AS cluster(\'test_shard_localhost\', \'default\', \'buffer\') CREATE TABLE default.url\n(\n `n` UInt64,\n `col` String\n)\nENGINE = URL(\'https://localhost:8443/?query=select+n,+_table+from+default.merge+format+CSV\', \'CSV\') -CREATE TABLE default.rich_syntax AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'default\', \'view\'))) +CREATE TABLE default.rich_syntax\n(\n `n` Int64\n) AS remote(\'localhos{x|y|t}\', cluster(\'test_shard_localhost\', remote(\'127.0.0.{1..4}\', \'default\', \'view\'))) CREATE VIEW default.view\n(\n `n` Int64\n) AS\nSELECT toInt64(n) AS n\nFROM \n(\n SELECT toString(n) AS n\n FROM default.merge\n WHERE _table != \'qwerty\'\n ORDER BY _table ASC\n)\nUNION ALL\nSELECT *\nFROM default.file CREATE DICTIONARY default.dict\n(\n `n` UInt64,\n `col` String DEFAULT \'42\'\n)\nPRIMARY KEY n\nSOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9440 SECURE 1 USER \'default\' TABLE \'url\'))\nLIFETIME(MIN 0 MAX 1)\nLAYOUT(CACHE(SIZE_IN_CELLS 1)) 16 diff --git a/tests/queries/0_stateless/014 b/tests/queries/0_stateless/014 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01457_create_as_table_function_structure.reference b/tests/queries/0_stateless/01457_create_as_table_function_structure.reference new file mode 100644 index 00000000000..7169648d524 --- /dev/null +++ b/tests/queries/0_stateless/01457_create_as_table_function_structure.reference @@ -0,0 +1,7 @@ +CREATE TABLE test_01457.tf_remote\n(\n `n` Int8\n) AS remote(\'localhost\', \'default\', \'tmp\') +CREATE TABLE test_01457.tf_remote_explicit_structure\n(\n `n` UInt64\n) AS remote(\'localhost\', \'default\', \'tmp\') +CREATE TABLE test_01457.tf_numbers\n(\n `number` String\n) AS numbers(1) +0 Int8 +0 Int8 +0 UInt64 +0 String diff --git a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql new file mode 100644 index 00000000000..1949c084960 --- /dev/null +++ b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql @@ -0,0 +1,28 @@ +DROP DATABASE IF EXISTS test_01457; + +CREATE DATABASE test_01457; + +CREATE TABLE tmp (n Int8) ENGINE=Memory; + +CREATE TABLE test_01457.tf_remote AS remote('localhost', currentDatabase(), 'tmp'); +SHOW CREATE TABLE test_01457.tf_remote; +CREATE TABLE test_01457.tf_remote_explicit_structure (n UInt64) AS remote('localhost', currentDatabase(), 'tmp'); +SHOW CREATE TABLE test_01457.tf_remote_explicit_structure; +CREATE TABLE test_01457.tf_numbers (number String) AS numbers(1); +SHOW CREATE TABLE test_01457.tf_numbers; + +DROP TABLE tmp; + +DETACH DATABASE test_01457; +ATTACH DATABASE test_01457; + +CREATE TABLE tmp (n Int8) ENGINE=Memory; +INSERT INTO test_01457.tf_remote_explicit_structure VALUES (0); -- { serverError 122 } +INSERT INTO test_01457.tf_remote VALUES (0); + +SELECT (*,).1 AS c, toTypeName(c) FROM tmp; +SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_remote; +SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_remote_explicit_structure; +SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_numbers; + +DROP DATABASE test_01457; From 41dd39382447ff4dd88f3af8c8fda72965af551a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 1 Sep 2020 20:55:35 +0300 Subject: [PATCH 033/441] fixes --- src/Databases/DatabaseOrdinary.cpp | 7 +++++-- src/Interpreters/ya.make | 1 + src/Storages/StorageTableFunction.h | 1 + .../0_stateless/01461_alter_table_function.reference | 9 ++++----- tests/queries/0_stateless/01461_alter_table_function.sql | 4 +--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 0512a155418..3f536f7c995 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -254,9 +254,12 @@ void DatabaseOrdinary::alterTable(const Context & context, const StorageID & tab auto & ast_create_query = ast->as(); - if (ast_create_query.as_table_function) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot alter table {} because it was created AS table function", backQuote(table_name)); + bool has_structure = ast_create_query.columns_list && ast_create_query.columns_list->columns; + if (ast_create_query.as_table_function && !has_structure) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot alter table {} because it was created AS table function" + " and doesn't have structure in metadata", backQuote(table_name)); + assert(has_structure); ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns); ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.secondary_indices); ASTPtr new_constraints = InterpreterCreateQuery::formatConstraints(metadata.constraints); diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 23cde61a744..cc3b63778f9 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -58,6 +58,7 @@ SRCS( ExtractExpressionInfoVisitor.cpp FillingRow.cpp getClusterName.cpp + getHeaderForProcessingStage.cpp getTableExpressions.cpp HashJoin.cpp IdentifierSemantic.cpp diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 87af3a49a5c..4a0afdd5fb1 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -37,6 +37,7 @@ public: auto nested_storage = get_nested(); nested_storage->startup(); + nested_storage->renameInMemory(getStorageID()); nested = nested_storage; get_nested = {}; return nested; diff --git a/tests/queries/0_stateless/01461_alter_table_function.reference b/tests/queries/0_stateless/01461_alter_table_function.reference index 395155967a9..b552dd81b77 100644 --- a/tests/queries/0_stateless/01461_alter_table_function.reference +++ b/tests/queries/0_stateless/01461_alter_table_function.reference @@ -1,7 +1,6 @@ -CREATE TABLE default.table_from_remote AS remote(\'localhost\', \'system\', \'numbers\') -CREATE TABLE default.table_from_remote AS remote(\'localhost\', \'system\', \'numbers\') -CREATE TABLE default.table_from_numbers AS numbers(1000) -CREATE TABLE default.table_from_numbers AS numbers(1000) +CREATE TABLE default.table_from_remote\n(\n `number` UInt64\n) AS remote(\'localhost\', \'system\', \'numbers\') +CREATE TABLE default.table_from_remote\n(\n `number` UInt64,\n `col` UInt8\n) AS remote(\'localhost\', \'system\', \'numbers\') +CREATE TABLE default.table_from_numbers\n(\n `number` UInt64\n) AS numbers(1000) +CREATE TABLE default.table_from_numbers\n(\n `number` UInt64\n) AS numbers(1000) CREATE TABLE default.table_from_select\n(\n `number` UInt64\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 CREATE TABLE default.table_from_select\n(\n `number` UInt64,\n `col` UInt8\n)\nENGINE = MergeTree()\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -1 diff --git a/tests/queries/0_stateless/01461_alter_table_function.sql b/tests/queries/0_stateless/01461_alter_table_function.sql index e242d1f0b7b..11f643f1e3e 100644 --- a/tests/queries/0_stateless/01461_alter_table_function.sql +++ b/tests/queries/0_stateless/01461_alter_table_function.sql @@ -6,7 +6,7 @@ CREATE TABLE table_from_remote AS remote('localhost', 'system', 'numbers'); SHOW CREATE TABLE table_from_remote; -ALTER TABLE table_from_remote ADD COLUMN col UInt8; --{serverError 48} +ALTER TABLE table_from_remote ADD COLUMN col UInt8; SHOW CREATE TABLE table_from_remote; @@ -26,8 +26,6 @@ ALTER TABLE table_from_select ADD COLUMN col UInt8; SHOW CREATE TABLE table_from_select; -SELECT 1; - DROP TABLE IF EXISTS table_from_remote; DROP TABLE IF EXISTS table_from_select; DROP TABLE IF EXISTS table_from_numbers; From ee7b8a797a7aed63eee854dedba69e9403fe4d48 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 2 Sep 2020 18:07:53 +0300 Subject: [PATCH 034/441] better code, fix Distributed format of StorageFile --- src/Interpreters/InterpreterCreateQuery.cpp | 6 +- src/Storages/StorageFile.cpp | 57 +++++++++++-------- src/Storages/StorageFile.h | 2 + src/Storages/StorageProxy.h | 1 - src/Storages/StorageTableFunction.h | 18 ++++-- src/TableFunctions/ITableFunction.cpp | 11 ++-- src/TableFunctions/ITableFunction.h | 16 +++++- src/TableFunctions/ITableFunctionFileLike.cpp | 11 +++- src/TableFunctions/ITableFunctionFileLike.h | 5 +- src/TableFunctions/ITableFunctionXDBC.cpp | 3 +- src/TableFunctions/ITableFunctionXDBC.h | 2 +- .../TableFunctionGenerateRandom.cpp | 5 +- .../TableFunctionGenerateRandom.h | 3 +- src/TableFunctions/TableFunctionInput.cpp | 4 +- src/TableFunctions/TableFunctionInput.h | 3 +- src/TableFunctions/TableFunctionMerge.cpp | 3 +- src/TableFunctions/TableFunctionMerge.h | 2 +- src/TableFunctions/TableFunctionMySQL.cpp | 3 +- src/TableFunctions/TableFunctionMySQL.h | 2 +- src/TableFunctions/TableFunctionNumbers.cpp | 3 +- src/TableFunctions/TableFunctionNumbers.h | 3 +- src/TableFunctions/TableFunctionRemote.cpp | 14 +++-- src/TableFunctions/TableFunctionRemote.h | 4 +- src/TableFunctions/TableFunctionS3.cpp | 3 +- src/TableFunctions/TableFunctionS3.h | 4 +- src/TableFunctions/TableFunctionValues.cpp | 3 +- src/TableFunctions/TableFunctionValues.h | 3 +- src/TableFunctions/TableFunctionZeros.cpp | 3 +- src/TableFunctions/TableFunctionZeros.h | 3 +- tests/queries/0_stateless/014 | 0 ...eate_as_table_function_structure.reference | 3 + ...457_create_as_table_function_structure.sql | 7 ++- 32 files changed, 131 insertions(+), 79 deletions(-) delete mode 100644 tests/queries/0_stateless/014 diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index c6a010e8240..20825f35109 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -495,8 +495,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS /// Table function without columns list. auto table_function = TableFunctionFactory::instance().get(create.as_table_function, context); properties.columns = table_function->getActualTableStructure(context); - if (properties.columns.empty()) //FIXME TableFunctionFile may return empty structure for Distributed format - return {}; + assert(!properties.columns.empty()); } else throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); @@ -583,6 +582,9 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { + if (create.as_table_function) + return; + if (create.storage || create.is_view || create.is_materialized_view || create.is_live_view || create.is_dictionary) { if (create.temporary && create.storage->engine->name != "Memory") diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 6ead7a914ff..fc777b68d60 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -52,6 +52,7 @@ namespace ErrorCodes extern const int UNKNOWN_IDENTIFIER; extern const int INCORRECT_FILE_NAME; extern const int FILE_DOESNT_EXIST; + extern const int INCOMPATIBLE_COLUMNS; } namespace @@ -126,11 +127,33 @@ void checkCreationIsAllowed(const Context & context_global, const std::string & } } +Strings StorageFile::getPathsList(const String & table_path, const String & user_files_path, const Context & context) +{ + String user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); + Poco::Path poco_path = Poco::Path(table_path); + if (poco_path.isRelative()) + poco_path = Poco::Path(user_files_absolute_path, poco_path); + + Strings paths; + const String path = poco_path.absolute().toString(); + if (path.find_first_of("*?{") == std::string::npos) + paths.push_back(path); + else + paths = listFilesWithRegexpMatching("/", path); + + for (const auto & cur_path : paths) + checkCreationIsAllowed(context, user_files_absolute_path, cur_path); + + return paths; +} + StorageFile::StorageFile(int table_fd_, CommonArguments args) : StorageFile(args) { if (args.context.getApplicationType() == Context::ApplicationType::SERVER) throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED); + if (args.format_name == "Distributed") + throw Exception("Distributed format is allowed only with explicit file path", ErrorCodes::INCORRECT_FILE_NAME); is_db_table = false; use_table_fd = true; @@ -145,36 +168,22 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us : StorageFile(args) { is_db_table = false; - std::string user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString(); - Poco::Path poco_path = Poco::Path(table_path_); - if (poco_path.isRelative()) - poco_path = Poco::Path(user_files_absolute_path, poco_path); - - const std::string path = poco_path.absolute().toString(); - if (path.find_first_of("*?{") == std::string::npos) - paths.push_back(path); - else - paths = listFilesWithRegexpMatching("/", path); - - for (const auto & cur_path : paths) - checkCreationIsAllowed(args.context, user_files_absolute_path, cur_path); + paths = getPathsList(table_path_, user_files_path, args.context); if (args.format_name == "Distributed") { if (paths.empty()) - { throw Exception("Cannot get table structure from file, because no files match specified name", ErrorCodes::INCORRECT_FILE_NAME); - } - else - { - auto & first_path = paths[0]; - Block header = StorageDistributedDirectoryMonitor::createStreamFromFile(first_path)->getHeader(); + auto & first_path = paths[0]; + Block header = StorageDistributedDirectoryMonitor::createStreamFromFile(first_path)->getHeader(); - StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(ColumnsDescription(header.getNamesAndTypesList())); - setInMemoryMetadata(storage_metadata); - } + StorageInMemoryMetadata storage_metadata; + auto columns = ColumnsDescription(header.getNamesAndTypesList()); + if (!args.columns.empty() && columns != args.columns) + throw Exception("Table structure and file structure are different", ErrorCodes::INCOMPATIBLE_COLUMNS); + storage_metadata.setColumns(columns); + setInMemoryMetadata(storage_metadata); } } @@ -183,6 +192,8 @@ StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArgu { if (relative_table_dir_path.empty()) throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); + if (args.format_name == "Distributed") + throw Exception("Distributed format is allowed only with explicit file path", ErrorCodes::INCORRECT_FILE_NAME); String table_dir_path = base_path + relative_table_dir_path + "/"; Poco::File(table_dir_path).createDirectories(); diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index ea70dcd5311..f331538b4c7 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -60,6 +60,8 @@ public: NamesAndTypesList getVirtuals() const override; + static Strings getPathsList(const String & table_path, const String & user_files_path, const Context & context); + protected: friend class StorageFileSource; friend class StorageFileBlockOutputStream; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 81edcf27c64..92200ff5775 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -6,7 +6,6 @@ namespace DB { - class StorageProxy : public IStorage { public: diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 4a0afdd5fb1..9b698cb3954 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -18,11 +18,14 @@ namespace ErrorCodes using GetNestedStorageFunc = std::function; +/// Lazily creates underlying storage. +/// Adds ConversionTransform in case of structure mismatch. class StorageTableFunctionProxy final : public StorageProxy { public: - StorageTableFunctionProxy(const StorageID & table_id_, GetNestedStorageFunc get_nested_, ColumnsDescription cached_columns) - : StorageProxy(table_id_), get_nested(std::move(get_nested_)) + StorageTableFunctionProxy(const StorageID & table_id_, GetNestedStorageFunc get_nested_, + ColumnsDescription cached_columns, bool add_conversion_ = true) + : StorageProxy(table_id_), get_nested(std::move(get_nested_)), add_conversion(add_conversion_) { StorageInMemoryMetadata cached_metadata; cached_metadata.setColumns(std::move(cached_columns)); @@ -80,10 +83,12 @@ public: cnames += c + " "; auto storage = getNested(); auto nested_metadata = storage->getInMemoryMetadataPtr(); - auto pipe = storage->read(column_names, nested_metadata, query_info, context, processed_stage, max_block_size, num_streams); - if (!pipe.empty()) + auto pipe = storage->read(column_names, nested_metadata, query_info, context, + processed_stage, max_block_size, num_streams); + if (!pipe.empty() && add_conversion) { - auto to_header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, query_info, context, processed_stage); + auto to_header = getHeaderForProcessingStage(*this, column_names, metadata_snapshot, + query_info, context, processed_stage); pipe.addSimpleTransform([&](const Block & header) { return std::make_shared( @@ -103,7 +108,7 @@ public: auto storage = getNested(); auto cached_structure = metadata_snapshot->getSampleBlock(); auto actual_structure = storage->getInMemoryMetadataPtr()->getSampleBlock(); - if (!blocksHaveEqualStructure(actual_structure, cached_structure)) + if (!blocksHaveEqualStructure(actual_structure, cached_structure) && add_conversion) { throw Exception("Source storage and table function have different structure", ErrorCodes::INCOMPATIBLE_COLUMNS); } @@ -126,6 +131,7 @@ private: mutable std::mutex nested_mutex; mutable GetNestedStorageFunc get_nested; mutable StoragePtr nested; + const bool add_conversion; }; } diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 13282275322..fd5cdf6e219 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -14,21 +14,24 @@ namespace ProfileEvents namespace DB { -StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const +StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & context, const std::string & table_name, + ColumnsDescription cached_columns) const { ProfileEvents::increment(ProfileEvents::TableFunctionExecute); context.checkAccess(AccessType::CREATE_TEMPORARY_TABLE | StorageFactory::instance().getSourceAccessType(getStorageTypeName())); bool no_conversion_required = hasStaticStructure() && cached_columns == getActualTableStructure(context); if (cached_columns.empty() || no_conversion_required) - return executeImpl(ast_function, context, table_name); + return executeImpl(ast_function, context, table_name, std::move(cached_columns)); auto get_storage = [=, tf = shared_from_this()]() -> StoragePtr { - return tf->executeImpl(ast_function, context, table_name); + return tf->executeImpl(ast_function, context, table_name, cached_columns); }; - return std::make_shared(StorageID(getDatabaseName(), table_name), std::move(get_storage), std::move(cached_columns)); + /// It will request actual table structure and create underlying storage lazily + return std::make_shared(StorageID(getDatabaseName(), table_name), std::move(get_storage), + std::move(cached_columns), needStructureConversion()); } } diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index b41213a5e09..4a73adbdf80 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -22,6 +22,15 @@ class Context; * Example: * SELECT count() FROM remote('example01-01-1', merge, hits) * - go to `example01-01-1`, in `merge` database, `hits` table. + * + * + * When creating table AS table_function(...) we probably don't know structure of the table + * and have to request if from remote server, because structure is required to create a Storage. + * To avoid failures on server startup, we write obtained structure to metadata file. + * So, table function may have two different columns lists: + * - cached_columns written to metadata + * - the list returned from getActualTableStructure(...) + * See StorageTableFunctionProxy. */ class ITableFunction : public std::enable_shared_from_this @@ -32,10 +41,15 @@ public: /// Get the main function name. virtual std::string getName() const = 0; + /// Returns true if we always know table structure when executing table function + /// (e.g. structure is specified in table function arguments) virtual bool hasStaticStructure() const { return false; } + /// Returns false if storage returned by table function supports type conversion (e.g. StorageDistributed) + virtual bool needStructureConversion() const { return true; } virtual void parseArguments(const ASTPtr & /*ast_function*/, const Context & /*context*/) {} + /// Returns actual table structure probably requested from remote server, may fail virtual ColumnsDescription getActualTableStructure(const Context & /*context*/) const = 0; /// Create storage according to the query. @@ -44,7 +58,7 @@ public: virtual ~ITableFunction() {} private: - virtual StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const = 0; + virtual StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const = 0; virtual const char * getStorageTypeName() const = 0; }; diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index b362a646f9e..f876da02fd1 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -9,6 +9,8 @@ #include #include +#include +#include #include #include @@ -20,6 +22,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int INCORRECT_FILE_NAME; } void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const Context & context) @@ -57,7 +60,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, const C compression_method = args[3]->as().value.safeGet(); } -StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr ITableFunctionFileLike::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto columns = getActualTableStructure(context); StoragePtr storage = getStorage(filename, format, columns, const_cast(context), table_name, compression_method); @@ -70,7 +73,11 @@ ColumnsDescription ITableFunctionFileLike::getActualTableStructure(const Context if (structure.empty()) { assert(getName() == "file" && format == "Distributed"); - return {}; /// TODO get matching path, read structure + Strings paths = StorageFile::getPathsList(filename, context.getUserFilesPath(), context); + if (paths.empty()) + throw Exception("Cannot get table structure from file, because no files match specified name", ErrorCodes::INCORRECT_FILE_NAME); + auto read_stream = StorageDistributedDirectoryMonitor::createStreamFromFile(paths[0]); + return ColumnsDescription{read_stream->getHeader().getNamesAndTypesList()}; } return parseColumnsListFromString(structure, context); } diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index 40111eebc5c..f1c648ac0aa 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB { @@ -14,7 +13,7 @@ class Context; class ITableFunctionFileLike : public ITableFunction { private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; virtual StoragePtr getStorage( const String & source, const String & format, const ColumnsDescription & columns, Context & global_context, @@ -24,6 +23,8 @@ private: void parseArguments(const ASTPtr & ast_function, const Context & context) override; + bool hasStaticStructure() const override { return true; } + String filename; String format; String structure; diff --git a/src/TableFunctions/ITableFunctionXDBC.cpp b/src/TableFunctions/ITableFunctionXDBC.cpp index 6ba62b8ebf1..67d1257fe4c 100644 --- a/src/TableFunctions/ITableFunctionXDBC.cpp +++ b/src/TableFunctions/ITableFunctionXDBC.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -85,7 +84,7 @@ ColumnsDescription ITableFunctionXDBC::getActualTableStructure(const Context & c return ColumnsDescription{columns}; } -StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr ITableFunctionXDBC::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { assert(helper); auto columns = getActualTableStructure(context); diff --git a/src/TableFunctions/ITableFunctionXDBC.h b/src/TableFunctions/ITableFunctionXDBC.h index 218f361c025..fb0a0fd1185 100644 --- a/src/TableFunctions/ITableFunctionXDBC.h +++ b/src/TableFunctions/ITableFunctionXDBC.h @@ -18,7 +18,7 @@ namespace DB class ITableFunctionXDBC : public ITableFunction { private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; /* A factory method to create bridge helper, that will assist in remote interaction */ virtual BridgeHelperPtr createBridgeHelper(Context & context, diff --git a/src/TableFunctions/TableFunctionGenerateRandom.cpp b/src/TableFunctions/TableFunctionGenerateRandom.cpp index c88cb3229b5..15c2c2bfa1f 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.cpp +++ b/src/TableFunctions/TableFunctionGenerateRandom.cpp @@ -13,9 +13,6 @@ #include #include -#include -#include - #include "registerTableFunctions.h" @@ -82,7 +79,7 @@ ColumnsDescription TableFunctionGenerateRandom::getActualTableStructure(const Co return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionGenerateRandom::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto columns = getActualTableStructure(context); auto res = StorageGenerateRandom::create(StorageID(getDatabaseName(), table_name), columns, max_array_length, max_string_length, random_seed); diff --git a/src/TableFunctions/TableFunctionGenerateRandom.h b/src/TableFunctions/TableFunctionGenerateRandom.h index 8c13f853a0d..1d8839ba6d4 100644 --- a/src/TableFunctions/TableFunctionGenerateRandom.h +++ b/src/TableFunctions/TableFunctionGenerateRandom.h @@ -13,8 +13,9 @@ class TableFunctionGenerateRandom : public ITableFunction public: static constexpr auto name = "generateRandom"; std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "GenerateRandom"; } ColumnsDescription getActualTableStructure(const Context & context) const override; diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index 6563d8f0b3a..41c41835434 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -7,10 +7,8 @@ #include #include #include -#include #include #include -#include #include #include "registerTableFunctions.h" @@ -45,7 +43,7 @@ ColumnsDescription TableFunctionInput::getActualTableStructure(const Context & c return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionInput::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionInput::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto storage = StorageInput::create(StorageID(getDatabaseName(), table_name), getActualTableStructure(context)); storage->startup(); diff --git a/src/TableFunctions/TableFunctionInput.h b/src/TableFunctions/TableFunctionInput.h index 9190fe600f6..5809d48a77c 100644 --- a/src/TableFunctions/TableFunctionInput.h +++ b/src/TableFunctions/TableFunctionInput.h @@ -15,9 +15,10 @@ class TableFunctionInput : public ITableFunction public: static constexpr auto name = "input"; std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "Input"; } ColumnsDescription getActualTableStructure(const Context & context) const override; diff --git a/src/TableFunctions/TableFunctionMerge.cpp b/src/TableFunctions/TableFunctionMerge.cpp index 6358a0bbbef..a878909e29d 100644 --- a/src/TableFunctions/TableFunctionMerge.cpp +++ b/src/TableFunctions/TableFunctionMerge.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -74,7 +73,7 @@ ColumnsDescription TableFunctionMerge::getActualTableStructure(const Context & c return ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}; } -StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto res = StorageMerge::create( StorageID(getDatabaseName(), table_name), diff --git a/src/TableFunctions/TableFunctionMerge.h b/src/TableFunctions/TableFunctionMerge.h index 2d038de32fb..a9c4dd6b038 100644 --- a/src/TableFunctions/TableFunctionMerge.h +++ b/src/TableFunctions/TableFunctionMerge.h @@ -16,7 +16,7 @@ public: static constexpr auto name = "merge"; std::string getName() const override { return name; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "Merge"; } ColumnsDescription getActualTableStructure(const Context & context) const override; diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index c957bb8c05f..ca67cdf2b24 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -13,7 +13,6 @@ # include # include # include -# include # include # include # include @@ -121,7 +120,7 @@ ColumnsDescription TableFunctionMySQL::getActualTableStructure(const Context & c return ColumnsDescription{columns}; } -StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { assert(!parsed_host_port.first.empty()); if (!pool) diff --git a/src/TableFunctions/TableFunctionMySQL.h b/src/TableFunctions/TableFunctionMySQL.h index f3c15a5fdd9..1fe5a4aa4ac 100644 --- a/src/TableFunctions/TableFunctionMySQL.h +++ b/src/TableFunctions/TableFunctionMySQL.h @@ -24,7 +24,7 @@ public: return name; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "MySQL"; } ColumnsDescription getActualTableStructure(const Context & context) const override; diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 84edd3935b3..984b26d5de4 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -28,7 +27,7 @@ ColumnsDescription TableFunctionNumbers::getActualTableStructure( } template -StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionNumbers::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { if (const auto * function = ast_function->as()) { diff --git a/src/TableFunctions/TableFunctionNumbers.h b/src/TableFunctions/TableFunctionNumbers.h index 5cbde6cf0a3..9c54125551f 100644 --- a/src/TableFunctions/TableFunctionNumbers.h +++ b/src/TableFunctions/TableFunctionNumbers.h @@ -17,8 +17,9 @@ class TableFunctionNumbers : public ITableFunction public: static constexpr auto name = multithreaded ? "numbers_mt" : "numbers"; std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "SystemNumbers"; } UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 594b47a3666..2e34e82ce36 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -45,7 +45,6 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, const Cont String cluster_description; String remote_database; String remote_table; - //ASTPtr remote_table_function_ptr; String username; String password; @@ -137,7 +136,6 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, const Cont for (auto ast : args) setIdentifierSpecial(ast); - //ClusterPtr cluster; if (!cluster_name.empty()) { /// Use an existing cluster from the main config @@ -190,18 +188,22 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, const Cont if (!remote_table_function_ptr && remote_table.empty()) throw Exception("The name of remote table cannot be empty", ErrorCodes::BAD_ARGUMENTS); - //auto remote_table_id = StorageID::createEmpty(); remote_table_id.database_name = remote_database; remote_table_id.table_name = remote_table; } -StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const { + /// StorageDistributed supports mismatching structure of remote table, so we can use outdated structure for CREATE ... AS remote(...) + /// without additional conversion in StorageTableFunctionProxy + if (cached_columns.empty()) + cached_columns = getActualTableStructure(context); + assert(cluster); StoragePtr res = remote_table_function_ptr ? StorageDistributed::create( StorageID(getDatabaseName(), table_name), - getActualTableStructure(context), + cached_columns, ConstraintsDescription{}, remote_table_function_ptr, String{}, @@ -213,7 +215,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, con cluster) : StorageDistributed::create( StorageID(getDatabaseName(), table_name), - getActualTableStructure(context), + cached_columns, ConstraintsDescription{}, remote_table_id.database_name, remote_table_id.table_name, diff --git a/src/TableFunctions/TableFunctionRemote.h b/src/TableFunctions/TableFunctionRemote.h index ff214bb7332..6ec591a34ac 100644 --- a/src/TableFunctions/TableFunctionRemote.h +++ b/src/TableFunctions/TableFunctionRemote.h @@ -24,8 +24,10 @@ public: ColumnsDescription getActualTableStructure(const Context & context) const override; + bool needStructureConversion() const override { return false; } + private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "Distributed"; } void parseArguments(const ASTPtr & ast_function, const Context & context) override; diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index a4f403f7875..dfe1cf6e792 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -63,7 +62,7 @@ ColumnsDescription TableFunctionS3::getActualTableStructure(const Context & cont return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { Poco::URI uri (filename); S3::URI s3_uri (uri); diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 45738de8ba8..722fb9eb23c 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -22,12 +22,14 @@ public: { return name; } + bool hasStaticStructure() const override { return true; } protected: StoragePtr executeImpl( const ASTPtr & ast_function, const Context & context, - const std::string & table_name) const override; + const std::string & table_name, + ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "S3"; } diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index 4e892c83247..2d01d581fda 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -95,7 +94,7 @@ ColumnsDescription TableFunctionValues::getActualTableStructure(const Context & return parseColumnsListFromString(structure, context); } -StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { auto columns = getActualTableStructure(context); diff --git a/src/TableFunctions/TableFunctionValues.h b/src/TableFunctions/TableFunctionValues.h index b66f5bb457f..549fa2de507 100644 --- a/src/TableFunctions/TableFunctionValues.h +++ b/src/TableFunctions/TableFunctionValues.h @@ -12,8 +12,9 @@ class TableFunctionValues : public ITableFunction public: static constexpr auto name = "values"; std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "Values"; } ColumnsDescription getActualTableStructure(const Context & context) const override; diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp index 5ba6c034e09..22bf47f5e9f 100644 --- a/src/TableFunctions/TableFunctionZeros.cpp +++ b/src/TableFunctions/TableFunctionZeros.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -27,7 +26,7 @@ ColumnsDescription TableFunctionZeros::getActualTableStructure(co } template -StoragePtr TableFunctionZeros::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +StoragePtr TableFunctionZeros::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const { if (const auto * function = ast_function->as()) { diff --git a/src/TableFunctions/TableFunctionZeros.h b/src/TableFunctions/TableFunctionZeros.h index c50a17d046d..2944e2c1c9d 100644 --- a/src/TableFunctions/TableFunctionZeros.h +++ b/src/TableFunctions/TableFunctionZeros.h @@ -17,8 +17,9 @@ class TableFunctionZeros : public ITableFunction public: static constexpr auto name = multithreaded ? "zeros_mt" : "zeros"; std::string getName() const override { return name; } + bool hasStaticStructure() const override { return true; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "SystemZeros"; } UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; diff --git a/tests/queries/0_stateless/014 b/tests/queries/0_stateless/014 deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/01457_create_as_table_function_structure.reference b/tests/queries/0_stateless/01457_create_as_table_function_structure.reference index 7169648d524..0deaa20171a 100644 --- a/tests/queries/0_stateless/01457_create_as_table_function_structure.reference +++ b/tests/queries/0_stateless/01457_create_as_table_function_structure.reference @@ -1,7 +1,10 @@ CREATE TABLE test_01457.tf_remote\n(\n `n` Int8\n) AS remote(\'localhost\', \'default\', \'tmp\') CREATE TABLE test_01457.tf_remote_explicit_structure\n(\n `n` UInt64\n) AS remote(\'localhost\', \'default\', \'tmp\') CREATE TABLE test_01457.tf_numbers\n(\n `number` String\n) AS numbers(1) +CREATE TABLE test_01457.tf_merge\n(\n `n` Int8\n) AS merge(\'default\', \'tmp\') +42 0 Int8 0 Int8 0 UInt64 0 String +0 Int8 diff --git a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql index 1949c084960..edfab747fc4 100644 --- a/tests/queries/0_stateless/01457_create_as_table_function_structure.sql +++ b/tests/queries/0_stateless/01457_create_as_table_function_structure.sql @@ -10,6 +10,8 @@ CREATE TABLE test_01457.tf_remote_explicit_structure (n UInt64) AS remote('local SHOW CREATE TABLE test_01457.tf_remote_explicit_structure; CREATE TABLE test_01457.tf_numbers (number String) AS numbers(1); SHOW CREATE TABLE test_01457.tf_numbers; +CREATE TABLE test_01457.tf_merge AS merge(currentDatabase(), 'tmp'); +SHOW CREATE TABLE test_01457.tf_merge; DROP TABLE tmp; @@ -17,12 +19,15 @@ DETACH DATABASE test_01457; ATTACH DATABASE test_01457; CREATE TABLE tmp (n Int8) ENGINE=Memory; -INSERT INTO test_01457.tf_remote_explicit_structure VALUES (0); -- { serverError 122 } +INSERT INTO test_01457.tf_remote_explicit_structure VALUES ('42'); +SELECT * FROM tmp; +TRUNCATE TABLE tmp; INSERT INTO test_01457.tf_remote VALUES (0); SELECT (*,).1 AS c, toTypeName(c) FROM tmp; SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_remote; SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_remote_explicit_structure; SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_numbers; +SELECT (*,).1 AS c, toTypeName(c) FROM test_01457.tf_merge; DROP DATABASE test_01457; From 4bf3f9e4a3bdd17e4f2e2fa35e971641ea5830cd Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 2 Sep 2020 18:22:53 +0300 Subject: [PATCH 035/441] use StorageProxy for StorageMaterializeMySQL --- src/Storages/StorageMaterializeMySQL.cpp | 2 +- src/Storages/StorageMaterializeMySQL.h | 20 +++++++++++++++----- src/Storages/StorageProxy.h | 1 + 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageMaterializeMySQL.cpp b/src/Storages/StorageMaterializeMySQL.cpp index 7d908736bdc..68981280619 100644 --- a/src/Storages/StorageMaterializeMySQL.cpp +++ b/src/Storages/StorageMaterializeMySQL.cpp @@ -23,7 +23,7 @@ namespace DB { StorageMaterializeMySQL::StorageMaterializeMySQL(const StoragePtr & nested_storage_, const DatabaseMaterializeMySQL * database_) - : IStorage(nested_storage_->getStorageID()), nested_storage(nested_storage_), database(database_) + : StorageProxy(nested_storage_->getStorageID()), nested_storage(nested_storage_), database(database_) { auto nested_memory_metadata = nested_storage->getInMemoryMetadata(); StorageInMemoryMetadata in_memory_metadata; diff --git a/src/Storages/StorageMaterializeMySQL.h b/src/Storages/StorageMaterializeMySQL.h index 4278ce64bd7..cadafaf778c 100644 --- a/src/Storages/StorageMaterializeMySQL.h +++ b/src/Storages/StorageMaterializeMySQL.h @@ -4,30 +4,40 @@ #if USE_MYSQL -#include +#include #include namespace DB { -class StorageMaterializeMySQL final : public ext::shared_ptr_helper, public IStorage +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + +class StorageMaterializeMySQL final : public ext::shared_ptr_helper, public StorageProxy { friend struct ext::shared_ptr_helper; public: String getName() const override { return "MaterializeMySQL"; } - bool supportsFinal() const override { return nested_storage->supportsFinal(); } - bool supportsSampling() const override { return nested_storage->supportsSampling(); } - StorageMaterializeMySQL(const StoragePtr & nested_storage_, const DatabaseMaterializeMySQL * database_); Pipe read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; + BlockOutputStreamPtr write(const ASTPtr &, const StorageMetadataPtr &, const Context &) override { throwNotAllowed(); } + NamesAndTypesList getVirtuals() const override; private: + StoragePtr getNested() const override { return nested_storage; } + [[noreturn]] void throwNotAllowed() const + { + throw Exception("This method is not allowed for MaterializeMySQ", ErrorCodes::NOT_IMPLEMENTED); + } + StoragePtr nested_storage; const DatabaseMaterializeMySQL * database; }; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 92200ff5775..7b010476b22 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB From 9d98effa5e166ff15a9bf61b6b76b7c72566f173 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 2 Sep 2020 20:39:37 +0300 Subject: [PATCH 036/441] write structure for TableFunctionView --- src/TableFunctions/TableFunctionView.cpp | 42 +++++++++++++----------- src/TableFunctions/TableFunctionView.h | 9 +++-- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp index 6166fa56f47..066b67de0fc 100644 --- a/src/TableFunctions/TableFunctionView.cpp +++ b/src/TableFunctions/TableFunctionView.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -16,25 +15,30 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -StoragePtr TableFunctionView::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +void TableFunctionView::parseArguments(const ASTPtr & ast_function, const Context & /*context*/) { - if (const auto * function = ast_function->as()) - { - if (function->query) - { - if (auto * select = function->query->as()) - { - auto sample = InterpreterSelectWithUnionQuery::getSampleBlock(function->query, context); - auto columns = ColumnsDescription(sample.getNamesAndTypesList()); - ASTCreateQuery create; - create.select = select; - auto res = StorageView::create(StorageID(getDatabaseName(), table_name), create, columns); - res->startup(); - return res; - } - } - } - throw Exception("Table function '" + getName() + "' requires a query argument.", ErrorCodes::BAD_ARGUMENTS); + const auto * function = ast_function->as(); + if (function && function->query && function->query->as()) + create.set(create.select, function->query); + else + throw Exception("Table function '" + getName() + "' requires a query argument.", ErrorCodes::BAD_ARGUMENTS); +} + +ColumnsDescription TableFunctionView::getActualTableStructure(const Context & context) const +{ + assert(create.select); + assert(create.children.size() == 1); + assert(create.children[0]->as()); + auto sample = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context); + return ColumnsDescription(sample.getNamesAndTypesList()); +} + +StoragePtr TableFunctionView::executeImpl(const ASTPtr & /*ast_function*/, const Context & context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const +{ + auto columns = getActualTableStructure(context); + auto res = StorageView::create(StorageID(getDatabaseName(), table_name), create, columns); + res->startup(); + return res; } void registerTableFunctionView(TableFunctionFactory & factory) diff --git a/src/TableFunctions/TableFunctionView.h b/src/TableFunctions/TableFunctionView.h index 49f51823735..f82d182d159 100644 --- a/src/TableFunctions/TableFunctionView.h +++ b/src/TableFunctions/TableFunctionView.h @@ -2,7 +2,7 @@ #include #include - +#include namespace DB { @@ -17,10 +17,13 @@ public: static constexpr auto name = "view"; std::string getName() const override { return name; } private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name, ColumnsDescription cached_columns) const override; const char * getStorageTypeName() const override { return "View"; } - UInt64 evaluateArgument(const Context & context, ASTPtr & argument) const; + void parseArguments(const ASTPtr & ast_function, const Context & context) override; + ColumnsDescription getActualTableStructure(const Context & context) const override; + + ASTCreateQuery create; }; From 9ba4613381942d920f26401e2c0c743771962698 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 1 Sep 2020 12:27:11 +0300 Subject: [PATCH 037/441] Excluding tests from fasttest runs since they depend on external libraries: OpenSSL --- docker/test/fasttest/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 1f8d612a125..abd2fd62c13 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -206,6 +206,8 @@ TESTS_TO_SKIP=( 01411_bayesian_ab_testing 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently 01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently + 01318_encrypt # Depends on OpenSSL + 01318_decrypt # Depends on OpenSSL # Not sure why these two fail even in sequential mode. Disabled for now # to make some progress. From af23a27f9f38accbf85eebdef79253fb81e512a3 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 3 Sep 2020 15:53:34 +0200 Subject: [PATCH 038/441] Reorganizing ldap tests. --- .../testflows/ldap/authentication/__init__.py | 0 .../{ => authentication}/configs/CA/ca.crt | 0 .../{ => authentication}/configs/CA/ca.key | 0 .../{ => authentication}/configs/CA/ca.srl | 0 .../configs/CA/dhparam.pem | 0 .../configs/CA/passphrase.txt | 0 .../configs/clickhouse/common.xml | 0 .../configs/clickhouse/config.d/logs.xml | 0 .../configs/clickhouse/config.d/ports.xml | 0 .../configs/clickhouse/config.d/remote.xml | 0 .../configs/clickhouse/config.d/ssl.xml | 0 .../configs/clickhouse/config.d/storage.xml | 0 .../configs/clickhouse/config.d/zookeeper.xml | 0 .../configs/clickhouse/config.xml | 12 +++++ .../configs/clickhouse/ssl/dhparam.pem | 0 .../configs/clickhouse/ssl/server.crt | 0 .../configs/clickhouse/ssl/server.key | 0 .../configs/clickhouse/users.xml | 0 .../configs/clickhouse1/config.d/macros.xml | 0 .../configs/clickhouse2/config.d/macros.xml | 0 .../configs/clickhouse3/config.d/macros.xml | 0 .../configs/ldap1/config/export.ldif | 0 .../configs/ldap2/certs/ca.crt | 0 .../configs/ldap2/certs/dhparam.pem | 0 .../configs/ldap2/certs/ldap.crt | 0 .../configs/ldap2/certs/ldap.csr | 0 .../configs/ldap2/certs/ldap.key | 0 .../configs/ldap2/config/export.ldif | 0 .../configs/ldap3/certs/ca.crt | 0 .../configs/ldap3/certs/dhparam.pem | 0 .../configs/ldap3/certs/ldap.crt | 0 .../configs/ldap3/certs/ldap.csr | 0 .../configs/ldap3/certs/ldap.key | 0 .../configs/ldap3/config/export.ldif | 0 .../configs/ldap4/certs/ca.crt | 0 .../configs/ldap4/certs/dhparam.pem | 0 .../configs/ldap4/certs/ldap.crt | 0 .../configs/ldap4/certs/ldap.csr | 0 .../configs/ldap4/certs/ldap.key | 0 .../configs/ldap4/config/export.ldif | 0 .../configs/ldap5/config/export.ldif | 0 .../configs/ldap5/ldap2/certs/ca.crt | 0 .../configs/ldap5/ldap2/certs/dhparam.pem | 0 .../configs/ldap5/ldap2/certs/ldap.crt | 0 .../configs/ldap5/ldap2/certs/ldap.csr | 0 .../configs/ldap5/ldap2/certs/ldap.key | 0 .../configs/ldap5/ldap2/config/export.ldif | 0 .../docker-compose/clickhouse-service.yml | 0 .../docker-compose/docker-compose.yml | 0 .../docker-compose/openldap-service.yml | 0 .../docker-compose/zookeeper-service.yml | 0 .../ldap/authentication/regression.py | 54 +++++++++++++++++++ .../requirements/__init__.py | 0 .../requirements/requirements.md | 0 .../requirements/requirements.py | 0 .../tests/authentications.py | 4 +- .../ldap/{ => authentication}/tests/common.py | 2 +- .../{ => authentication}/tests/connections.py | 4 +- .../tests/multiple_servers.py | 4 +- .../ldap/{ => authentication}/tests/sanity.py | 2 +- .../tests/server_config.py | 4 +- .../{ => authentication}/tests/user_config.py | 4 +- tests/testflows/ldap/regression.py | 46 +++------------- 63 files changed, 84 insertions(+), 52 deletions(-) create mode 100644 tests/testflows/ldap/authentication/__init__.py rename tests/testflows/ldap/{ => authentication}/configs/CA/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/ca.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/ca.srl (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/passphrase.txt (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/common.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/logs.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/ports.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/remote.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/ssl.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/storage.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/zookeeper.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.xml (97%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/ssl/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/ssl/server.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/ssl/server.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/users.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse1/config.d/macros.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse2/config.d/macros.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse3/config.d/macros.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap1/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/clickhouse-service.yml (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/docker-compose.yml (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/openldap-service.yml (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/zookeeper-service.yml (100%) create mode 100755 tests/testflows/ldap/authentication/regression.py rename tests/testflows/ldap/{ => authentication}/requirements/__init__.py (100%) rename tests/testflows/ldap/{ => authentication}/requirements/requirements.md (100%) rename tests/testflows/ldap/{ => authentication}/requirements/requirements.py (100%) rename tests/testflows/ldap/{ => authentication}/tests/authentications.py (99%) rename tests/testflows/ldap/{ => authentication}/tests/common.py (99%) rename tests/testflows/ldap/{ => authentication}/tests/connections.py (98%) rename tests/testflows/ldap/{ => authentication}/tests/multiple_servers.py (88%) rename tests/testflows/ldap/{ => authentication}/tests/sanity.py (95%) rename tests/testflows/ldap/{ => authentication}/tests/server_config.py (99%) rename tests/testflows/ldap/{ => authentication}/tests/user_config.py (98%) diff --git a/tests/testflows/ldap/authentication/__init__.py b/tests/testflows/ldap/authentication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/ldap/configs/CA/ca.crt b/tests/testflows/ldap/authentication/configs/CA/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/CA/ca.crt rename to tests/testflows/ldap/authentication/configs/CA/ca.crt diff --git a/tests/testflows/ldap/configs/CA/ca.key b/tests/testflows/ldap/authentication/configs/CA/ca.key similarity index 100% rename from tests/testflows/ldap/configs/CA/ca.key rename to tests/testflows/ldap/authentication/configs/CA/ca.key diff --git a/tests/testflows/ldap/configs/CA/ca.srl b/tests/testflows/ldap/authentication/configs/CA/ca.srl similarity index 100% rename from tests/testflows/ldap/configs/CA/ca.srl rename to tests/testflows/ldap/authentication/configs/CA/ca.srl diff --git a/tests/testflows/ldap/configs/CA/dhparam.pem b/tests/testflows/ldap/authentication/configs/CA/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/CA/dhparam.pem rename to tests/testflows/ldap/authentication/configs/CA/dhparam.pem diff --git a/tests/testflows/ldap/configs/CA/passphrase.txt b/tests/testflows/ldap/authentication/configs/CA/passphrase.txt similarity index 100% rename from tests/testflows/ldap/configs/CA/passphrase.txt rename to tests/testflows/ldap/authentication/configs/CA/passphrase.txt diff --git a/tests/testflows/ldap/configs/clickhouse/common.xml b/tests/testflows/ldap/authentication/configs/clickhouse/common.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/common.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/common.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/logs.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/logs.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/logs.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/logs.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/ports.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/ports.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/ports.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/ports.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/remote.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/remote.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/remote.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/remote.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/ssl.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/ssl.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/ssl.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/ssl.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/storage.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/storage.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/storage.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/storage.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/zookeeper.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/zookeeper.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/zookeeper.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml similarity index 97% rename from tests/testflows/ldap/configs/clickhouse/config.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.xml index d34d2c35253..80c3150d326 100644 --- a/tests/testflows/ldap/configs/clickhouse/config.xml +++ b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml @@ -120,6 +120,18 @@ /var/lib/clickhouse/access/ + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + users.xml diff --git a/tests/testflows/ldap/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/ldap/authentication/configs/clickhouse/ssl/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/ssl/dhparam.pem rename to tests/testflows/ldap/authentication/configs/clickhouse/ssl/dhparam.pem diff --git a/tests/testflows/ldap/configs/clickhouse/ssl/server.crt b/tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.crt similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/ssl/server.crt rename to tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.crt diff --git a/tests/testflows/ldap/configs/clickhouse/ssl/server.key b/tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.key similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/ssl/server.key rename to tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.key diff --git a/tests/testflows/ldap/configs/clickhouse/users.xml b/tests/testflows/ldap/authentication/configs/clickhouse/users.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/users.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/users.xml diff --git a/tests/testflows/ldap/configs/clickhouse1/config.d/macros.xml b/tests/testflows/ldap/authentication/configs/clickhouse1/config.d/macros.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse1/config.d/macros.xml rename to tests/testflows/ldap/authentication/configs/clickhouse1/config.d/macros.xml diff --git a/tests/testflows/ldap/configs/clickhouse2/config.d/macros.xml b/tests/testflows/ldap/authentication/configs/clickhouse2/config.d/macros.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse2/config.d/macros.xml rename to tests/testflows/ldap/authentication/configs/clickhouse2/config.d/macros.xml diff --git a/tests/testflows/ldap/configs/clickhouse3/config.d/macros.xml b/tests/testflows/ldap/authentication/configs/clickhouse3/config.d/macros.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse3/config.d/macros.xml rename to tests/testflows/ldap/authentication/configs/clickhouse3/config.d/macros.xml diff --git a/tests/testflows/ldap/configs/ldap1/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap1/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap1/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap1/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap2/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap2/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap2/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap2/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap2/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap2/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap2/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap2/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap2/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap2/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap2/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap2/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap3/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap3/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap3/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap3/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap3/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap3/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap3/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap3/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap3/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap3/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap3/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap3/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap4/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap4/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap4/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap4/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap4/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap4/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap4/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap4/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap4/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap5/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap5/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap5/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap5/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/config/export.ldif diff --git a/tests/testflows/ldap/docker-compose/clickhouse-service.yml b/tests/testflows/ldap/authentication/docker-compose/clickhouse-service.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/clickhouse-service.yml rename to tests/testflows/ldap/authentication/docker-compose/clickhouse-service.yml diff --git a/tests/testflows/ldap/docker-compose/docker-compose.yml b/tests/testflows/ldap/authentication/docker-compose/docker-compose.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/docker-compose.yml rename to tests/testflows/ldap/authentication/docker-compose/docker-compose.yml diff --git a/tests/testflows/ldap/docker-compose/openldap-service.yml b/tests/testflows/ldap/authentication/docker-compose/openldap-service.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/openldap-service.yml rename to tests/testflows/ldap/authentication/docker-compose/openldap-service.yml diff --git a/tests/testflows/ldap/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/zookeeper-service.yml rename to tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py new file mode 100755 index 00000000000..9d0a5ca743f --- /dev/null +++ b/tests/testflows/ldap/authentication/regression.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..", "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from ldap.authentication.requirements import * + +# Cross-outs of known fails +xfails = { + "connection protocols/tls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls require cert default demand": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls with custom port": + [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], + "connection protocols/tls cipher suite": + [(Fail, "can't get it to work")] +} + +@TestFeature +@Name("authentication") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS_007_LDAP_Authentication("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path): + """ClickHouse integration with LDAP regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) + Scenario(run=load("ldap.authentication.tests.multiple_servers", "scenario")) + Feature(run=load("ldap.authentication.tests.connections", "feature")) + Feature(run=load("ldap.authentication.tests.server_config", "feature")) + Feature(run=load("ldap.authentication.tests.user_config", "feature")) + Feature(run=load("ldap.authentication.tests.authentications", "feature")) + +if main(): + regression() diff --git a/tests/testflows/ldap/requirements/__init__.py b/tests/testflows/ldap/authentication/requirements/__init__.py similarity index 100% rename from tests/testflows/ldap/requirements/__init__.py rename to tests/testflows/ldap/authentication/requirements/__init__.py diff --git a/tests/testflows/ldap/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md similarity index 100% rename from tests/testflows/ldap/requirements/requirements.md rename to tests/testflows/ldap/authentication/requirements/requirements.md diff --git a/tests/testflows/ldap/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py similarity index 100% rename from tests/testflows/ldap/requirements/requirements.py rename to tests/testflows/ldap/authentication/requirements/requirements.py diff --git a/tests/testflows/ldap/tests/authentications.py b/tests/testflows/ldap/authentication/tests/authentications.py similarity index 99% rename from tests/testflows/ldap/tests/authentications.py rename to tests/testflows/ldap/authentication/tests/authentications.py index a1fb27bd51a..1b21dce7cc1 100644 --- a/tests/testflows/ldap/tests/authentications.py +++ b/tests/testflows/ldap/authentication/tests/authentications.py @@ -4,8 +4,8 @@ import random from multiprocessing.dummy import Pool from testflows.core import * from testflows.asserts import error -from ldap.tests.common import * -from ldap.requirements import * +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * servers = { "openldap1": { diff --git a/tests/testflows/ldap/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py similarity index 99% rename from tests/testflows/ldap/tests/common.py rename to tests/testflows/ldap/authentication/tests/common.py index eb21923e930..1e382f8942c 100644 --- a/tests/testflows/ldap/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -172,7 +172,7 @@ def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-serve return Config(content, path, name, uid, "users.xml") def add_users_identified_with_ldap(*users): - """Add one or more users that are identified via + """Add one or more users that are identified via an ldap server using RBAC. """ node = current().context.node diff --git a/tests/testflows/ldap/tests/connections.py b/tests/testflows/ldap/authentication/tests/connections.py similarity index 98% rename from tests/testflows/ldap/tests/connections.py rename to tests/testflows/ldap/authentication/tests/connections.py index 8de4b3f4d01..f16f6c29b0e 100644 --- a/tests/testflows/ldap/tests/connections.py +++ b/tests/testflows/ldap/authentication/tests/connections.py @@ -1,8 +1,8 @@ from testflows.core import * from testflows.asserts import error -from ldap.tests.common import login -from ldap.requirements import * +from ldap.authentication.tests.common import login +from ldap.authentication.requirements import * @TestScenario @Requirements( diff --git a/tests/testflows/ldap/tests/multiple_servers.py b/tests/testflows/ldap/authentication/tests/multiple_servers.py similarity index 88% rename from tests/testflows/ldap/tests/multiple_servers.py rename to tests/testflows/ldap/authentication/tests/multiple_servers.py index aefc0116fa2..6e906023b0a 100644 --- a/tests/testflows/ldap/tests/multiple_servers.py +++ b/tests/testflows/ldap/authentication/tests/multiple_servers.py @@ -1,8 +1,8 @@ from testflows.core import * from testflows.asserts import error -from ldap.tests.common import login -from ldap.requirements import RQ_SRS_007_LDAP_Authentication_MultipleServers +from ldap.authentication.tests.common import login +from ldap.authentication.requirements import RQ_SRS_007_LDAP_Authentication_MultipleServers @TestScenario @Name("multiple servers") diff --git a/tests/testflows/ldap/tests/sanity.py b/tests/testflows/ldap/authentication/tests/sanity.py similarity index 95% rename from tests/testflows/ldap/tests/sanity.py rename to tests/testflows/ldap/authentication/tests/sanity.py index 9e5d8a2ddd7..542fa2a48b1 100644 --- a/tests/testflows/ldap/tests/sanity.py +++ b/tests/testflows/ldap/authentication/tests/sanity.py @@ -1,7 +1,7 @@ from testflows.core import * from testflows.asserts import error -from ldap.tests.common import add_user_to_ldap, delete_user_from_ldap +from ldap.authentication.tests.common import add_user_to_ldap, delete_user_from_ldap @TestScenario @Name("sanity") diff --git a/tests/testflows/ldap/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py similarity index 99% rename from tests/testflows/ldap/tests/server_config.py rename to tests/testflows/ldap/authentication/tests/server_config.py index f3d03434afe..5658f7a9399 100644 --- a/tests/testflows/ldap/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -1,7 +1,7 @@ from testflows.core import * -from ldap.tests.common import * -from ldap.requirements import * +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * @TestScenario @Requirements( diff --git a/tests/testflows/ldap/tests/user_config.py b/tests/testflows/ldap/authentication/tests/user_config.py similarity index 98% rename from tests/testflows/ldap/tests/user_config.py rename to tests/testflows/ldap/authentication/tests/user_config.py index edc85a5877e..7927d72b510 100644 --- a/tests/testflows/ldap/tests/user_config.py +++ b/tests/testflows/ldap/authentication/tests/user_config.py @@ -2,8 +2,8 @@ import xml.etree.ElementTree as xmltree from testflows.core import * -from ldap.tests.common import * -from ldap.requirements import * +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * @TestScenario @Requirements( diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 567807fc0a8..8520941ed0b 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -2,53 +2,19 @@ import sys from testflows.core import * -append_path(sys.path, "..") +append_path(sys.path, "..") -from helpers.cluster import Cluster from helpers.argparser import argparser -from ldap.requirements import * -# Cross-outs of known fails -xfails = { - "connection protocols/tls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls require cert default demand": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls with custom port": - [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], - "connection protocols/tls cipher suite": - [(Fail, "can't get it to work")] -} - -@TestFeature -@Name("ldap authentication") +@TestModule +@Name("ldap") @ArgumentParser(argparser) -@Requirements( - RQ_SRS_007_LDAP_Authentication("1.0") -) -@XFails(xfails) def regression(self, local, clickhouse_binary_path): - """ClickHouse integration with LDAP regression module. + """ClickHouse LDAP integration regression module. """ - nodes = { - "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), - } - - with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: - self.context.cluster = cluster + args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} - Scenario(run=load("ldap.tests.sanity", "scenario")) - Scenario(run=load("ldap.tests.multiple_servers", "scenario")) - Feature(run=load("ldap.tests.connections", "feature")) - Feature(run=load("ldap.tests.server_config", "feature")) - Feature(run=load("ldap.tests.user_config", "feature")) - Feature(run=load("ldap.tests.authentications", "feature")) + Feature(test=load("ldap.authentication.regression", "regression"))(**args) if main(): regression() From 85b913deaac7ac1bb36b5c06a118da5974a626f7 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 3 Sep 2020 16:53:09 +0200 Subject: [PATCH 039/441] Adding regression tests for LDAP external user directory. --- tests/testflows/helpers/cluster.py | 5 +- .../ldap/external_user_directory/__init__.py | 0 .../external_user_directory/configs/CA/ca.crt | 22 + .../external_user_directory/configs/CA/ca.key | 30 + .../external_user_directory/configs/CA/ca.srl | 1 + .../configs/CA/dhparam.pem | 8 + .../configs/CA/passphrase.txt | 1 + .../configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 107 ++ .../configs/clickhouse/config.d/ssl.xml | 17 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../configs/clickhouse/config.xml | 448 +++++++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../configs/clickhouse/users.xml | 133 ++ .../configs/clickhouse1/config.d/macros.xml | 8 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../configs/ldap1/config/export.ldif | 64 + .../configs/ldap2/certs/ca.crt | 22 + .../configs/ldap2/certs/dhparam.pem | 5 + .../configs/ldap2/certs/ldap.crt | 20 + .../configs/ldap2/certs/ldap.csr | 17 + .../configs/ldap2/certs/ldap.key | 27 + .../configs/ldap2/config/export.ldif | 64 + .../configs/ldap3/certs/ca.crt | 22 + .../configs/ldap3/certs/dhparam.pem | 5 + .../configs/ldap3/certs/ldap.crt | 20 + .../configs/ldap3/certs/ldap.csr | 17 + .../configs/ldap3/certs/ldap.key | 27 + .../configs/ldap3/config/export.ldif | 64 + .../configs/ldap4/certs/ca.crt | 22 + .../configs/ldap4/certs/dhparam.pem | 5 + .../configs/ldap4/certs/ldap.crt | 20 + .../configs/ldap4/certs/ldap.csr | 17 + .../configs/ldap4/certs/ldap.key | 27 + .../configs/ldap4/config/export.ldif | 64 + .../configs/ldap5/config/export.ldif | 64 + .../configs/ldap5/ldap2/certs/ca.crt | 22 + .../configs/ldap5/ldap2/certs/dhparam.pem | 5 + .../configs/ldap5/ldap2/certs/ldap.crt | 20 + .../configs/ldap5/ldap2/certs/ldap.csr | 17 + .../configs/ldap5/ldap2/certs/ldap.key | 27 + .../configs/ldap5/ldap2/config/export.ldif | 64 + .../docker-compose/clickhouse-service.yml | 28 + .../docker-compose/docker-compose.yml | 162 +++ .../docker-compose/openldap-service.yml | 40 + .../docker-compose/zookeeper-service.yml | 18 + .../external_user_directory/regression.py | 55 + .../requirements/__init__.py | 1 + .../requirements/requirements.py | 1133 +++++++++++++++++ .../tests/authentications.py | 509 ++++++++ .../external_user_directory/tests/common.py | 181 +++ .../tests/connections.py | 270 ++++ .../tests/external_user_directory_config.py | 283 ++++ .../external_user_directory/tests/roles.py | 314 +++++ .../tests/server_config.py | 305 +++++ .../external_user_directory/tests/simple.py | 24 + tests/testflows/ldap/regression.py | 1 + 63 files changed, 4979 insertions(+), 2 deletions(-) create mode 100644 tests/testflows/ldap/external_user_directory/__init__.py create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/ca.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/ca.srl create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/ldap/external_user_directory/regression.py create mode 100644 tests/testflows/ldap/external_user_directory/requirements/__init__.py create mode 100644 tests/testflows/ldap/external_user_directory/requirements/requirements.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/authentications.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/common.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/connections.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/roles.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/server_config.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/simple.py diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 8288e700e3b..761acce40d3 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -53,7 +53,7 @@ class ClickHouseNode(Node): continue assert False, "container is not healthy" - def restart(self, timeout=120, safe=True): + def restart(self, timeout=120, safe=True, wait_healthy=True): """Restart node. """ if safe: @@ -73,7 +73,8 @@ class ClickHouseNode(Node): self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) - self.wait_healthy(timeout) + if wait_healthy: + self.wait_healthy(timeout) def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, raise_on_exception=False, step=By, settings=None, *args, **kwargs): diff --git a/tests/testflows/ldap/external_user_directory/__init__.py b/tests/testflows/ldap/external_user_directory/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/ca.crt b/tests/testflows/ldap/external_user_directory/configs/CA/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/ca.key b/tests/testflows/ldap/external_user_directory/configs/CA/ca.key new file mode 100644 index 00000000000..e7a7f664dcf --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/ca.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,D06B9754A2069EBB4E77065DC9B605A1 + +FJT794Z6AUuUB5Vp5W2iR6zzCvQUg2dtKoE+xhFdbgC7lmSfA2W/O9fx15Il67Yj +Bbpm9Y6yteUSDQpJrvBdkhXeBkYEa5K1CA+0Jdx98nzwP3KBhHNxVVrTWRc5kniB +LMV3iBQEbAafxgL7gN+EWr3eV7w7ZSqT7D5br/mlBALU62gv2UzwTXLu1CgyNWMC +HIPjIX50Zga+BnhZhtQvM4Yj1gOsn+X6AaEZ3KjTfCDqthYQf2ldswW4gAlPAq83 ++INq9Spx+QG97Z+1XO2DmmGTZL0z+OFLT+3y26/UcftM26ODY09Dcf3gt0n6RIUV +0KsD1qQL0ppu4CHVnbIkOKMBe86qBl+kG8FVmyhgZ8D9ULlF1tpyTVKvHR82V2B5 +ztbc5EY1Fhb+r7OVVJlbCeo/bWmWybODZrpN49x5gGZpM3+8ApaHupGZ+cRFkQKG +rDpqC5gflT3WwFNxtP5noWcV+Gzb3riXNM3c8G5aIpLZwmmaTLK9ahKqMcq4Ljf+ +hir8kuCMqIKt3m7Ceoj4wAHSP8xO0y/cc1WYNb3CI0Emk795aR6IPUw4vDEXHG27 +OLoCJTvl/JKRWJGkdQx8wKAs/uw/qwtbhYoQJccTjfvy4NXH3tpSgxCE8OTWuEch +TAN8ra1PDGAUu+1MeT5gZ9uI1BEU6hXMME4mVRpJdcmw9MVy3V+B6rkUqX3kFAfR +e2ueF5qgIp+A4/UlVe5cKdWAQxu4BnUESLooA7cbgcLypdao9bRx9bXH8S3aNgxW +IdgICpc/v8wAX2yqMe191KgR9Vh1p0RCw/kEGVgWfY/IaQpsaYuq5quZbvr/fN5T +d++ySAMaPysaCadLUdZJLw56uk4Y+PYzR+ygjTX9dCCHedrAU8RYM55FJ/fyD3bQ +Hn9/n7PZyWy6u/TYt6dhlcYxaS3Opzw4eAQB8tGZJRYQ3AKpHpTEC57lXoMnUPKo ++nBmb0+YulylMZdns0WIBJlcv6qzIaNhDMrjyi18n1ezzPIGH7ivUjoXy2FL23q5 +f3aqJK4UUDEDkC8IeZkS+ykYxnohjFDhUyBe5gjryLqdMdy9EerehCWPf425AztX +c/EWPzDl46qmxWhugOlz3Fiw95VlYu0MUDRayHuZiYPplgJypChuU4EHJ+q8V2z3 +BwjSo1bD4nfc8f68qEOtdZ1u/ClcolMwlZQYDJz/DiE4JOcd2Gx4QSF5vaInm0/4 +mMj/ZWna4DAYFbH8IGh7xUPDqeIWhBYlgrD69ajKyay5Vu3La/d2QW20BhX35Ro2 +ZJVR+lfioMmxn4y481H2pv+5gOlGwh02Oa8qLhZBb8W+DvFShNk6mk87eCForFFT +CDgmvfsC/cS2wZkcFTecq6vbjFlt+OF13NCKlcO3wCm44D+bwVPeMrU6HycCVQw7 +SASrnP/th5sJbv11byb2lKgVdVHWk090bqnDwB9H2hGIb9JnPC9ZpaL/mocYyzTi +H9fcBrMYkL09FJGr3Uff7qEY4XQTMlLadXue3iKd19PRgV8cRyKp37MYI9/3iLwv +eYHLtMfrifZahf1ksOPeBphnlfzWo9qqfooUCaGxfSlNPUHhrHZ4aMiRyTE8Xeh2 +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/ca.srl b/tests/testflows/ldap/external_user_directory/configs/CA/ca.srl new file mode 100644 index 00000000000..66feb9c8a35 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/ca.srl @@ -0,0 +1 @@ +227B125D27B6B1A4B5955361365DF8EC2D7098C1 diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem new file mode 100644 index 00000000000..554d75696ee --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA1iatTn4jdw1WIu09qeLj8OEeLhzG/w2lI4RUeJT9nU+WTwegpvLN +/MvrIMIKHRmItyxgraYFau2moC7RKm7OKLmFt6e34QeMvM1vXpuwQav6mfp8GsYL +mEIw5riFcB73E32NN3g7qmfmurkTF28BohmqhuQp2et7FNoGBKQ6ePZzGHWil3yG +nEnCwyK0o3eP2IEytx2N50uUWVdfg3MN34L3wqpUivArrjBkoMpqm3/V3wdfoYG9 +ZQkH0gIxT/2FIixCLGlfBsJ1qA/Apz1BJZbGqVu5M5iiQmq+LWN5JLS3xYai4wJL +rIY8DhjbciSNVWkwTJHzaLwIQa9a6p6mUwIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt b/tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt new file mode 100644 index 00000000000..2cf58b2364c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt @@ -0,0 +1 @@ +altinity diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml new file mode 100644 index 00000000000..df952b28c82 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + 0.0.0.0 + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 +
+
diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..51be2a6e8e3 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,107 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..ca65ffd5e04 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,17 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml new file mode 100644 index 00000000000..80c3150d326 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml @@ -0,0 +1,448 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + +
diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif new file mode 100644 index 00000000000..621dd32ca0c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user1,ou=users,dc=company,dc=com +dn: cn=user1,ou=users,dc=company,dc=com +cn: user1 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user1 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user1 +uidnumber: 1101 +userpassword: user1 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif new file mode 100644 index 00000000000..6766aaae6f1 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user2,ou=users,dc=company,dc=com +dn: cn=user2,ou=users,dc=company,dc=com +cn: user2 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user2 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user2 +uidnumber: 1002 +userpassword: user2 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif new file mode 100644 index 00000000000..6ac9a995efd --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user3,ou=users,dc=company,dc=com +dn: cn=user3,ou=users,dc=company,dc=com +cn: user3 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user3 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user3 +uidnumber: 1003 +userpassword: user3 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif new file mode 100644 index 00000000000..36afdb4e350 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user4,ou=users,dc=company,dc=com +dn: cn=user4,ou=users,dc=company,dc=com +cn: user4 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user4 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user4 +uidnumber: 1004 +userpassword: user4 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif new file mode 100644 index 00000000000..bc3d2ff75fc --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user5,ou=users,dc=company,dc=com +dn: cn=user5,ou=users,dc=company,dc=com +cn: user5 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user5 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user5 +uidnumber: 1005 +userpassword: user5 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif new file mode 100644 index 00000000000..c6470176a5e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user1,ou=users,dc=company,dc=com +dn: cn=user1,ou=users,dc=company,dc=com +cn: user1 +gidnumber: 501 +givenname: John1 +homedirectory: /home/users/user1 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User1 +uid: user1 +uidnumber: 1001 +userpassword: user1 diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml new file mode 100644 index 00000000000..2a56876c72e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml b/tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..c8ff683df58 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml @@ -0,0 +1,162 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml new file mode 100644 index 00000000000..139907c513c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml @@ -0,0 +1,40 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + container_name: phpldapadmin + environment: + PHPLDAPADMIN_HTTPS=false: + ports: + - "8080:80" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py new file mode 100755 index 00000000000..0082e37d98e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..", "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from ldap.external_user_directory.requirements import * + +# Cross-outs of known fails +xfails = { + "connection protocols/tls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls require cert default demand": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls with custom port": + [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], + "connection protocols/tls cipher suite": + [(Fail, "can't get it to work")] +} + +@TestFeature +@Name("external user directory") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path): + """ClickHouse LDAP external user directory regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) + Scenario(run=load("ldap.external_user_directory.tests.simple", "scenario")) + Feature(run=load("ldap.external_user_directory.tests.server_config", "feature")) + Feature(run=load("ldap.external_user_directory.tests.external_user_directory_config", "feature")) + Feature(run=load("ldap.external_user_directory.tests.connections", "feature")) + Feature(run=load("ldap.external_user_directory.tests.authentications", "feature")) + Feature(run=load("ldap.external_user_directory.tests.roles", "feature")) + +if main(): + regression() diff --git a/tests/testflows/ldap/external_user_directory/requirements/__init__.py b/tests/testflows/ldap/external_user_directory/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py new file mode 100644 index 00000000000..7e3ced037fa --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -0,0 +1,1133 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.200827.1211600. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Requirement + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server\n' + 'as soon as they are added to the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow authentication of users that\n' + 'were previously defined only on the [LDAP] server but were removed\n' + 'from the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' + 'user name and password match [LDAP] server records for the user\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' + 'do not match [LDAP] server records for the user\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' + 'on the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' + 'for the user is changed on the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users after server is restarted\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authentication of valid users and\n' + 'prohibit authentication of invalid users using [LDAP] server\n' + 'in parallel without having invalid attempts affecting valid authentications\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol\n' + 'while connecting to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol\n' + 'while connecting to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' + 'plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_Validation = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support certificate validation used for [TLS] connections\n' + 'to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_SelfSigned = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support self-signed certificates for [TLS] connections\n' + 'to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_SpecificCertificationAuthority = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections\n' + 'to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_Anonymous = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_Unauthenticated = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL lookup user presence in the following priority:\n' + '\n' + '1. access control\n' + '2. `users.xml`\n' + '3. LDAP\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reject authentication attempt if any of the roles that are specified in the configuration\n' + 'of the external user directory are not defined at the time of the authentication attempt\n' + 'with an exception that if a user was able to authenticate in past and its internal user object was created and cached\n' + 'then the user SHALL be able to authenticate again, even if one of the roles is missing.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP\n' + 'users authenticated using external user directory if it is removed\n' + 'including currently cached users that are still able to authenticated where the removed\n' + 'role is specified in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reassign the role and add the privileges provided by the role\n' + 'when it is re-added after removal for all LDAP users authenticated using external user directory\n' + 'including any cached users where the re-added role was specified in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP\n' + 'users authenticated using external user directory unless the role is specified\n' + 'in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory\n' + 'including cached users when new privilege is added to one of the roles specified\n' + 'in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory\n' + 'including cached users when privilege is removed from all the roles specified\n' + 'in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support using the [LDAP] servers defined in the\n' + '`ldap_servers` section of the `config.xml` as the server to be used\n' + 'for a external user directory that uses an [LDAP] server as a source of user definitions.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support empty string as a server name.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' + 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify the prefix\n' + 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify the suffix\n' + 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Value = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' + '\n' + "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' + 'to enable SSL/TLS `ldaps://` protocol.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' + 'plain text `ldap://` protocol.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' + 'SSL/TLS `ldaps://` protocol.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' + 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify\n' + 'the minimum protocol version of SSL/TLS.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' + 'as a value of the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' + 'certificate verification behavior.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' + 'provided, the session SHALL be immediately terminated.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no\n' + 'certificate is provided, the session SHALL proceed normally.\n' + 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no certificate is provided, the session\n' + 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' + 'immediately terminated.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' + 'disable requesting of client certificate.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' + '[ClickHouse] to establish connection with the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSKeyFile = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' + 'specified by the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify to a path to\n' + 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertFile = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' + '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n' + 'The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' + '```\n' + '\n' + 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' + '[ClickHouse] and therefore might change.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n' + 'configuration file or of any configuration file inside the `config.d` directory.\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' localhost\n' + ' 636\n' + ' cn=\n' + ' , ou=users, dc=example, dc=com\n' + ' yes\n' + ' tls1.2\n' + ' demand\n' + ' /path/to/tls_cert_file\n' + ' /path/to/tls_key_file\n' + ' /path/to/tls_ca_cert_file\n' + ' /path/to/tls_ca_cert_dir\n' + ' ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' + ' \n' + '\n' + '```\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml`\n' + 'that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only use the first `` sub-section in the `` section of the `config.xml`\n' + 'if more than one `` sub-sections are present.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Syntax = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` section with the following syntax\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' \n' + ' my_ldap_server\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + '\n' + '```\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `server` parameter in the `` sub-section in the ``\n' + 'section of the `config.xml` that SHALL specify one of LDAP server names\n' + 'defined in `` section.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n' + 'is empty.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n' + 'is missing.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the ``\n' + 'if more than one `server` parameter is defined in the configuration.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the server specified as the value of the ``\n' + 'parameter is not defined.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the ``\n' + 'section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL\n' + 'be assigned to all users retrieved from the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only use the first definitition of the `roles` parameter\n' + 'in the `` sub-section in the ``\n' + 'if more than one `roles` parameter is defined in the configuration.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the role specified in the ``\n' + 'parameter does not exist locally.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n' + 'to perform any action if the `roles` parameter in the `` sub-section in the ``\n' + 'section is empty.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n' + 'to perform any action if the `roles` parameter in the `` sub-section in the ``\n' + 'section is missing.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support authenticating users with empty username\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8 = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support authenticating users with empty passwords\n' + 'even if an empty password is valid for the user and\n' + 'is allowed by the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support long password of at least 256 bytes\n' + 'that can be used to authenticate users when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8 = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' + 'used to authenticate users when using [LDAP] external user directory.\n' + ), + link=None + ) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py new file mode 100644 index 00000000000..76e55996f21 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -0,0 +1,509 @@ +# -*- coding: utf-8 -*- +import random + +from multiprocessing.dummy import Pool +from testflows.core import * +from testflows.asserts import error + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "never", + } +} + +@TestStep(When) +@Name("I login as {username} and execute query") +def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True): + self.context.node.query("SELECT 1", + settings=[("user", username), ("password", password)], + exitcode=exitcode or 0, + message=message, steps=steps) + +@TestOutline +def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None): + """Add user to LDAP and ClickHouse and then try to login.""" + self.context.ldap_node = self.context.cluster.node(server) + + if ch_user is None: + ch_user = {} + if login is None: + login = {} + if user is None: + user = {"cn": "myuser", "userpassword": "myuser"} + + with ldap_user(**user) as user: + username = login.get("username", user["cn"]) + password = login.get("password", user["userpassword"]) + + login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login(self, server, user_count=10, timeout=200): + """Check that login of valid and invalid LDAP authenticated users works in parallel.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + + with ldap_users(*users): + def login_with_valid_username_and_password(users, i, iterations=10): + with When(f"valid users try to login #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + + def login_with_valid_username_and_invalid_password(users, i, iterations=10): + with When(f"users try to login with valid username and invalid password #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], + password=(random_user["userpassword"] + randomword(1)), + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + + def login_with_invalid_username_and_valid_password(users, i, iterations=10): + with When(f"users try to login with invalid username and valid password #{i}"): + for i in range(iterations): + random_user = dict(users[random.randint(0, len(users)-1)]) + random_user["cn"] += randomword(1) + login_and_execute_query(username=random_user["cn"], + password=random_user["userpassword"], + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + + with When("I login in parallel"): + p = Pool(15) + tasks = [] + for i in range(5): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + with Then("it should work"): + for task in tasks: + task.get(timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers("1.0") +) +def login_after_user_is_added_to_ldap(self, server): + """Check that user can login as soon as it is added to LDAP.""" + user = {"cn": "myuser", "userpassword": "myuser"} + + with When(f"I add user to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers("1.0") +) +def login_after_user_is_deleted_from_ldap(self, server): + """Check that login fails after user is deleted from LDAP.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I delete this user from LDAP"): + delete_user_from_ldap(user) + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged("1.0") +) +def login_after_user_password_changed_in_ldap(self, server): + """Check that login fails after user password is changed in LDAP.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + + with And("when I try to login with the new password it should work"): + login_and_execute_query(username=user["cn"], password="newpassword") + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged("1.0") +) +def login_after_user_cn_changed_in_ldap(self, server): + """Check that login fails after user cn is changed in LDAP.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + new_user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I change user password in LDAP"): + new_user = change_user_cn_in_ldap(user, "myuser2") + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart("1.0") +) +def login_after_ldap_server_is_restarted(self, server, timeout=60): + """Check that login succeeds after LDAP server is restarted.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I restart LDAP server"): + self.context.ldap_node.restart() + + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart("1.0") +) +def login_after_clickhouse_server_is_restarted(self, server, timeout=60): + """Check that login succeeds after ClickHouse server is restarted.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I restart ClickHouse server"): + self.context.node.restart() + + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0") +) +def valid_username_with_valid_empty_password(self, server): + """Check that we can't login using valid username that has empty password.""" + user = {"cn": "empty_password", "userpassword": ""} + exitcode = 4 + message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0") +) +def valid_username_and_invalid_empty_password(self, server): + """Check that we can't login using valid username but invalid empty password.""" + username = "user_non_empty_password" + user = {"cn": username, "userpassword": username} + login = {"password": ""} + + exitcode = 4 + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0") +) +def valid_username_and_password(self, server): + """Check that we can login using valid username and password.""" + username = "valid_username_and_password" + user = {"cn": username, "userpassword": username} + + with When(f"I add user {username} to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def valid_username_and_password_invalid_server(self, server=None): + """Check that we can't login using valid username and valid + password but for a different server.""" + self.context.ldap_node = self.context.cluster.node("openldap1") + + user = {"username": "user2", "userpassword": "user2", "server": "openldap1"} + + exitcode = 4 + message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long("1.0"), +) +def valid_long_username_and_short_password(self, server): + """Check that we can login using valid very long username and short password.""" + username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" + user = {"cn": username, "userpassword": "long_username"} + + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def invalid_long_username_and_valid_short_password(self, server): + """Check that we can't login using slightly invalid long username but valid password.""" + username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" + user = {"cn": username, "userpassword": "long_username"} + login = {"username": f"{username}?"} + + exitcode = 4 + message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long("1.0") +) +def valid_short_username_and_long_password(self, server): + """Check that we can login using valid short username with very long password.""" + username = "long_password" + user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def valid_short_username_and_invalid_long_password(self, server): + """Check that we can't login using valid short username and invalid long password.""" + username = "long_password" + user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + login = {"password": user["userpassword"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def valid_username_and_invalid_password(self, server): + """Check that we can't login using valid username and invalid password.""" + username = "valid_username_and_invalid_password" + user = {"cn": username, "userpassword": username} + login = {"password": user["userpassword"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def invalid_username_and_valid_password(self, server): + """Check that we can't login using slightly invalid username but valid password.""" + username = "invalid_username_and_valid_password" + user = {"cn": username, "userpassword": username} + login = {"username": user["cn"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8("1.0") +) +def valid_utf8_username_and_ascii_password(self, server): + """Check that we can login using valid utf-8 username with ascii password.""" + username = "utf8_username_Gãńdåłf_Thê_Gręât" + user = {"cn": username, "userpassword": "utf8_username"} + + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8("1.0") +) +def valid_ascii_username_and_utf8_password(self, server): + """Check that we can login using valid ascii username with utf-8 password.""" + username = "utf8_password" + user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} + + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +def empty_username_and_empty_password(self, server=None): + """Check that we can login using empty username and empty password as + it will use the default user and that has an empty password.""" + login_and_execute_query(username="", password="") + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("1.0") +) +def user_lookup_priority(self, server): + """Check that users are looked up in the same priority + as they are defined in the `` section + of the `config.xml`. For this test we have the following priority list + as defined by the configuration files: + + * users.xml + * local directory + * LDAP external user directory + """ + self.context.ldap_node = self.context.cluster.node(server) + + message="DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + exitcode = 4 + + users = { + "default": {"username": "default", "password": "userdefault"}, + "local": {"username": "local", "password": "userlocal"}, + "ldap": {"username": "ldap", "password": "userldap"} + } + + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): + with rbac_users("local"): + with When("I try to login as 'default' user which is also defined in users.xml it should fail"): + login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) + + with When("I try to login as 'local' user which is also defined in local storage it should fail"): + login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) + + with When("I try to login as 'ldap' user defined only in LDAP it should work"): + login_and_execute_query(**users["ldap"]) + + +@TestOutline(Feature) +@Name("user authentications") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword("1.0") +) +def feature(self, servers=None, server=None, node="clickhouse1"): + """Check that users can be authenticated using an LDAP external user directory. + """ + self.context.node = self.context.cluster.node(node) + + if servers is None: + servers = globals()["servers"] + + if server is None: + server = "openldap1" + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + for scenario in loads(current_module(), Scenario): + scenario(server=server) diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py new file mode 100644 index 00000000000..4c7877035c8 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -0,0 +1,181 @@ +import os +import time +from contextlib import contextmanager + +import testflows.settings as settings +from testflows.core import * +from testflows.asserts import error +from ldap.authentication.tests.common import getuid, Config, ldap_servers, add_config +from ldap.authentication.tests.common import xmltree, xml_indent, xml_append, xml_with_utf8 +from ldap.authentication.tests.common import ldap_user, ldap_users, add_user_to_ldap, delete_user_from_ldap +from ldap.authentication.tests.common import change_user_password_in_ldap, change_user_cn_in_ldap +from ldap.authentication.tests.common import randomword + +@contextmanager +def table(name, create_statement, on_cluster=False): + node = current().context.node + try: + with Given(f"I have a {name} table"): + node.query(create_statement.format(name=name)) + yield name + finally: + with Finally("I drop the table"): + if on_cluster: + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") + else: + node.query(f"DROP TABLE IF EXISTS {name}") + +@contextmanager +def rbac_users(*users): + node = current().context.node + try: + with Given("I have local users"): + for user in users: + with By(f"creating user {user}"): + node.query(f"CREATE USER OR REPLACE {user} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user}'") + yield users + finally: + with Finally("I drop local users"): + for user in users: + with By(f"dropping user {user}", flags=TE): + node.query(f"DROP USER IF EXISTS {user}") + +@contextmanager +def rbac_roles(*roles): + node = current().context.node + try: + with Given("I have roles"): + for role in roles: + with By(f"creating role {role}"): + node.query(f"CREATE ROLE OR REPLACE {role}") + yield roles + finally: + with Finally("I drop the roles"): + for role in roles: + with By(f"dropping role {role}", flags=TE): + node.query(f"DROP ROLE IF EXISTS {role}") + +def create_ldap_external_user_directory_config_content(server=None, roles=None, **kwargs): + """Create LDAP external user directory configuration file content. + """ + return create_entries_ldap_external_user_directory_config_content(entries=[([server], [roles])], **kwargs) + +def create_entries_ldap_external_user_directory_config_content(entries, config_d_dir="/etc/clickhouse-server/config.d", + config_file="ldap_external_user_directories.xml"): + """Create configurattion file content that contains + one or more entries for the LDAP external user directory. + + For example, + + ```xml + + + my_ldap_server + my_user + + + ``` + """ + uid = getuid() + path = os.path.join(config_d_dir, config_file) + name = config_file + + root = xmltree.fromstring("") + xml_user_directories = root.find("user_directories") + xml_user_directories.append(xmltree.Comment(text=f"LDAP external user directories {uid}")) + + for entry in entries: + servers, roles_entries = entry + xml_directory = xmltree.Element("ldap") + for server in servers: + if server is not None: + xml_append(xml_directory, "server", server) + if roles_entries: + for roles_entry in roles_entries: + xml_roles = xmltree.Element("roles") + if roles_entry: + for role in roles_entry: + if role is not None: + xml_append(xml_roles, role, "") + xml_directory.append(xml_roles) + xml_user_directories.append(xml_directory) + + xml_indent(root) + content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + + return Config(content, path, name, uid, "config.xml") + +def invalid_ldap_external_user_directory_config(server, roles, message, tail=20, timeout=20, config=None): + """Check that ClickHouse errors when trying to load invalid LDAP external user directory + configuration file. + """ + cluster = current().context.cluster + node = current().context.node + + if config is None: + config = create_ldap_external_user_directory_config_content(server=server, roles=roles) + + try: + with Given("I prepare the error log by writting empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with When("I add the config", description=config.path): + command = f"cat < {config.path}\n{config.content}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + with When("I restart ClickHouse to apply the config changes"): + node.restart(safe=False, wait_healthy=False) + + finally: + with Finally(f"I remove {config.name}"): + with By("removing invalid configuration file"): + system_config_path = os.path.join(current_dir(), "..", "configs", node.name, "config.d", config.path.split("config.d/")[-1]) + cluster.command(None, f'rm -rf {system_config_path}', timeout=timeout, exitcode=0) + + with And("restarting the node"): + node.restart(safe=False) + + with Then("error log should contain the expected error message"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + +@contextmanager +def ldap_external_user_directory(server, roles, config_d_dir="/etc/clickhouse-server/config.d", + config_file=None, timeout=20, restart=True, config=None): + """Add LDAP external user directory. + """ + if config_file is None: + config_file = f"ldap_external_user_directory_{getuid()}.xml" + if config is None: + config = create_ldap_external_user_directory_config_content(server=server, roles=roles, config_d_dir=config_d_dir, config_file=config_file) + return add_config(config, restart=restart) + +def login(servers, directory_server, *users, config=None): + """Configure LDAP server and LDAP external user directory and + try to login and execute a query""" + with ldap_servers(servers): + with rbac_roles(f"role_{getuid()}") as roles: + with ldap_external_user_directory(server=servers[directory_server]["host"], roles=roles, restart=True, config=config): + for user in users: + if user.get("login", False): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=user.get("exitcode", None), + message=user.get("message", None)) diff --git a/tests/testflows/ldap/external_user_directory/tests/connections.py b/tests/testflows/ldap/external_user_directory/tests/connections.py new file mode 100644 index 00000000000..ba734bb6c71 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/connections.py @@ -0,0 +1,270 @@ +from testflows.core import * +from testflows.asserts import error + +from ldap.external_user_directory.tests.common import login +from ldap.external_user_directory.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0") +) +def plain_text(self): + """Check that we can perform LDAP user authentication using `plain text` connection protocol. + """ + servers = { + "openldap1": { + "host": "openldap1", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + ] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") +) +def plain_text_with_custom_port(self): + """Check that we can perform LDAP user authentication using `plain text` connection protocol + with the server that uses custom port. + """ + servers = { + "openldap3": { + "host": "openldap3", + "port": "3089", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap3", "username": "user3", "password": "user3", "login": True} + ] + login(servers, "openldap3", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") +) +def tls_with_custom_port(self): + """Check that we can perform LDAP user authentication using `TLS` connection protocol + with the server that uses custom port. + """ + servers = { + "openldap4": { + "host": "openldap4", + "port": "6036", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap4", "username": "user4", "password": "user4", "login": True} + ] + login(servers, "openldap4", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") +) +def starttls_with_custom_port(self): + """Check that we can perform LDAP user authentication using `StartTLS` connection protocol + with the server that uses custom port. + """ + servers = { + "openldap4": { + "host": "openldap4", + "port": "3089", + "enable_tls": "starttls", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap4", "username": "user4", "password": "user4", "login": True} + ] + login(servers, "openldap4", *users) + +def tls_connection(enable_tls, tls_require_cert): + """Try to login using LDAP user authentication over a TLS connection.""" + servers = { + "openldap2": { + "host": "openldap2", + "enable_tls": enable_tls, + "tls_require_cert": tls_require_cert, + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + + requirements = [] + + if tls_require_cert == "never": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never("1.0")] + elif tls_require_cert == "allow": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow("1.0")] + elif tls_require_cert == "try": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try("1.0")] + elif tls_require_cert == "demand": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand("1.0")] + + with Example(name=f"tls_require_cert='{tls_require_cert}'", requirements=requirements): + login(servers, "openldap2", *users) + +@TestScenario +@Examples("enable_tls tls_require_cert", [ + ("yes", "never"), + ("yes", "allow"), + ("yes", "try"), + ("yes", "demand") +]) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default("1.0") +) +def tls(self): + """Check that we can perform LDAP user authentication using `TLS` connection protocol. + """ + for example in self.examples: + tls_connection(*example) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default("1.0") +) +def tls_enable_tls_default_yes(self): + """Check that the default value for the `enable_tls` is set to `yes`.""" + servers = { + "openldap2": { + "host": "openldap2", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default("1.0") +) +def tls_require_cert_default_demand(self): + """Check that the default value for the `tls_require_cert` is set to `demand`.""" + servers = { + "openldap2": { + "host": "openldap2", + "enable_tls": "yes", + "port": "636", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + login(servers, "openldap2", *users) + +@TestScenario +@Examples("enable_tls tls_require_cert", [ + ("starttls", "never"), + ("starttls", "allow"), + ("starttls", "try"), + ("starttls", "demand") +]) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0") +) +def starttls(self): + """Check that we can perform LDAP user authentication using legacy `StartTLS` connection protocol. + """ + for example in self.examples: + tls_connection(*example) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite("1.0") +) +def tls_cipher_suite(self): + """Check that `tls_cipher_suite` parameter can be used specify allowed cipher suites.""" + servers = { + "openldap4": { + "host": "openldap4", + "port": "6036", + "tls_require_cert": "never", + "tls_cipher_suite": "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC", + "tls_minimum_protocol_version": "tls1.2", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap4", "username": "user4", "password": "user4", "login": True} + ] + login(servers, "openldap4", *users) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values("1.0") +) +@Examples("version exitcode message", [ + ("ssl2", None, None), + ("ssl3", None, None), + ("tls1.0", None, None), + ("tls1.1", None, None), + ("tls1.2", None, None) +]) +def tls_minimum_protocol_version(self, version, exitcode, message): + """Check that `tls_minimum_protocol_version` parameter can be used specify + to specify the minimum protocol version of SSL/TLS.""" + + servers = { + "openldap4": { + "host": "openldap4", + "port": "6036", + "tls_require_cert": "never", + "tls_minimum_protocol_version": version, + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + + users = [{ + "server": "openldap4", "username": "user4", "password": "user4", + "login": True, "exitcode": int(exitcode) if exitcode is not None else None, "message": message + }] + + login(servers,"openldap4", *users) + +@TestFeature +@Name("connection protocols") +def feature(self, node="clickhouse1"): + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py new file mode 100644 index 00000000000..d29d0124dc2 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -0,0 +1,283 @@ +from testflows.core import * + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("1.0") +) +def more_than_one_user_directory(self, timeout=20): + """Check when more than one LDAP user directory is + defined inside a configuration file. + """ + message = "DB::Exception: Duplicate storage type 'ldap' at user_directories" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + "openldap2": { + "host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "never" + } + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + role = f"role_{getuid()}" + entries = [ + (["openldap1"], [(role,)]), + (["openldap2"], [(role,)]) + ] + + with ldap_servers(servers): + with rbac_roles(role) as roles: + config = create_entries_ldap_external_user_directory_config_content(entries) + invalid_ldap_external_user_directory_config(server=None, roles=None, message=message, timeout=timeout, config=config) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty("1.0") +) +def empty_server(self, timeout=20): + """Check that empty string in a `server` field is not allowed. + """ + message = "DB::Exception: Empty 'server' field for LDAP user directory" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + + with ldap_servers(servers): + with rbac_roles(f"role_{getuid()}") as roles: + invalid_ldap_external_user_directory_config(server="", roles=roles, message=message, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing("1.0") +) +def missing_server(self, timeout=20): + """Check that missing `server` field is not allowed. + """ + message = "DB::Exception: Missing 'server' field for LDAP user directory" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + + with ldap_servers(servers): + with rbac_roles(f"role_{getuid()}") as roles: + invalid_ldap_external_user_directory_config(server=None, roles=roles, message=message, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne("1.0") +) +def defined_twice_server(self, timeout=20): + """Check that when `server` field is defined twice that only the first + entry is used. + """ + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + + role = f"role_{getuid()}" + entries = [ + (["openldap1", "openldap2"], [(role,)]) + ] + + with ldap_servers(servers): + with rbac_roles(role) as roles: + config = create_entries_ldap_external_user_directory_config_content(entries) + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])]) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid("1.0") +) +def invalid_server(self, timeout=20): + """Check when `server` field value is invalid. + """ + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + role = f"role_{getuid()}" + + entries = [ + (["openldap2"], [(role,)]) + ] + + with ldap_servers(servers): + with rbac_roles(role) as roles: + config = create_entries_ldap_external_user_directory_config_content(entries) + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=36, message="DB::Exception: LDAP server 'openldap2' is not configured") + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty("1.0") +) +def empty_roles(self, timeout=20): + """Check when `roles` parameter is empty then user can't read any tables. + """ + message = "DB::Exception: user1: Not enough privileges." + exitcode = 241 + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + + entries = [ + (["openldap1"], [[]]) + ] + + with ldap_servers(servers): + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + config = create_entries_ldap_external_user_directory_config_content(entries) + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne("1.0") +) +def defined_twice_roles(self, timeout=20): + """Check that when `roles` is defined twice then only the first entry is used. + """ + node = self.context.node + + create_statement = "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + roles = [f"role0_{getuid()}", f"role1_{getuid()}"] + entries = [ + (["openldap1"], [[roles[0]],[roles[1]]]) + ] + + with ldap_servers(servers): + with rbac_roles(*roles): + with table(f"table0_{getuid()}", create_statement) as table0_name, \ + table(f"table1_{getuid()}", create_statement) as table1_name: + + with Given("I grant select privilege for the first table to the first role"): + node.query(f"GRANT SELECT ON {table0_name} TO {roles[0]}") + + with And("I grant select privilege for the second table to the second role"): + node.query(f"GRANT SELECT ON {table1_name} TO {roles[1]}") + + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and try to read from the first table"): + current().context.node.query(f"SELECT * FROM {table0_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])]) + + with And(f"I login as {user['username']} again and try to read from the second table"): + current().context.node.query(f"SELECT * FROM {table0_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])]) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid("1.0") +) +def invalid_role_in_roles(self, timeout=20): + """Check that an error is returned when LDAP users try to authenticate + if an invalid role is specified inside the `roles` section. + """ + exitcode = 4 + message = "DB::Exception: user1: Authentication failed" + + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + + with ldap_servers(servers): + with ldap_external_user_directory("openldap1", roles=["foo"], restart=True): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing("1.0") +) +def missing_roles(self, timeout=20): + """Check that when the `roles` are missing then + LDAP users can still login but can't read from any table. + """ + message = "DB::Exception: user1: Not enough privileges." + exitcode = 241 + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + entries = [ + (["openldap1"], None) + ] + + with ldap_servers(servers): + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=exitcode, message=message) + +@TestFeature +@Name("external user directory config") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Syntax("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition("1.0") +) +def feature(self, node="clickhouse1"): + """Check LDAP external user directory configuration. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/tests/testflows/ldap/external_user_directory/tests/roles.py b/tests/testflows/ldap/external_user_directory/tests/roles.py new file mode 100644 index 00000000000..d621993078b --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/roles.py @@ -0,0 +1,314 @@ +from testflows.core import * + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New("1.0") +) +def new_role(self, server, timeout=20): + """Check that new roles can't be assigned to any LDAP user + authenticated using external user directory. + """ + node = self.context.node + uid = getuid() + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and execute query simple query to cache the LDAP user"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with rbac_roles(f"new_role0_{uid}") as new_roles: + + message = "DB::Exception: Cannot update user `{user}` in ldap because this storage is readonly" + exitcode = 239 + + with And("I try to grant new role to the cached LDAP user"): + node.query(f"GRANT {new_roles[0]} TO {users[0]['username']}", + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + message = "DB::Exception: There is no role `{user}` in user directories" + exitcode = 255 + + with And("I try to grant new role to the non-cached LDAP user"): + node.query(f"GRANT {new_roles[0]} TO {users[1]['username']}", + exitcode=exitcode, message=message.format(user=users[1]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege("1.0") +) +def add_privilege(self, server, timeout=20): + """Check that we can add privilege to a role used + in the external user directory configuration. + """ + node = self.context.node + uid = getuid() + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and execute query that requires no privileges"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I login and try to read from the table without having select privilege"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with And(f"I login again and expect that cached LDAP user can successfully read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I login again and expect that non-cached LDAP user can successfully read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege("1.0") +) +def remove_privilege(self, server, timeout=20): + """Check that we can remove privilege from a role used + in the external user directory configuration. + """ + node = self.context.node + uid = getuid() + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login then LDAP user should be able to read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with When(f"I revoke select privilege from all the roles assigned to LDAP users"): + node.query(f"REVOKE SELECT ON {table_name} FROM {roles[0]}") + + with When(f"I login again then cached LDAP user should not be able to read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + with When(f"I login with non-cached LDAP user then the user should also not be able to read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])], + exitcode=exitcode, message=message.format(user=users[1]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed("1.0") +) +def remove_role(self, server, timeout=20): + """Check that when a role used in the external user directory configuration + is dynamically removed then any non-cached LDAP users should not be authenticated using + LDAP external user directory. + """ + node = self.context.node + uid = getuid() + exitcode = 4 + message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with When(f"I login and execute query that requires no privileges"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I remove one of the roles"): + node.query(f"DROP ROLE IF EXISTS {roles[1]}") + + with And(f"I try to login using cached LDAP user"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I try to login again using non-cached LDAP user"): + node.query(f"SELECT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])], + exitcode=exitcode, message=message.format(user=users[1]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges("1.0") +) +def remove_privilege_by_removing_role(self, server, timeout=20): + """Check that when the role used in the external user directory configuration + is dynamically removed then privileges are removed from all + LDAP users that are authenticated using external user directory. + """ + node = self.context.node + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + uid = getuid() + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and expect that LDAP user can read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I remove the role that grants the privilege"): + node.query(f"DROP ROLE IF EXISTS {roles[0]}") + + with And(f"I try to relogin and expect that cached LDAP user can login " + "but does not have privilege that was provided by the removed role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges("1.0") +) +def readd_privilege_by_readding_role(self, server, timeout=20): + """Check that when the role used in the external user directory configuration + is dynamically removed then all the privileges are removed from any + LDAP users authenticated using external user directory but when the role is re-added + then privileges are restored and non-cached users can login again. + """ + node = self.context.node + uid = getuid() + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and expect that LDAP user can read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I remove the role that grants the privilege"): + node.query(f"DROP ROLE IF EXISTS {roles[0]}") + + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + + with And(f"I try to relogin and expect that cached LDAP user can login " + "but does not have privilege that was provided by the removed role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + exitcode = 4 + + with And(f"I try to login using non-cached LDAP user and expect it to fail"): + node.query(f"SELECT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])], + exitcode=exitcode, message=message.format(user=users[1]["username"])) + + with When("I re-add the role"): + node.query(f"CREATE ROLE IF NOT EXISTS {roles[0]}") + + with And(f"I grant select privilege to the re-added role"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with And(f"I try to relogin and expect that cached LDAP user can login " + "and again has the privilege that is provided by the role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I try to login using non-cached LDAP expect it to work " + "with user also having privilege provided by the role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + +@TestFeature +@Name("roles") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles("1.0") +) +def feature(self, node="clickhouse1"): + """Check that all the users that are authenticated using + LDAP external user directory are assigned the roles specified + in the configuration of the LDAP external user directory. + """ + self.context.node = self.context.cluster.node(node) + + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + + with ldap_servers(servers): + for scenario in loads(current_module(), Scenario): + scenario(server="openldap1") diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py new file mode 100644 index 00000000000..b02b70eb6a7 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -0,0 +1,305 @@ +import time + +from testflows.core import * + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +from ldap.authentication.tests.common import invalid_server_config + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name("1.0") +) +def empty_server_name(self, timeout=20): + """Check that empty string as a server name is not allowed. + """ + servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + invalid_server_config(servers, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer("1.0") +) +def invalid_host(self): + """Check that server returns an error when LDAP server + host name is invalid. + """ + servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server" + }] + login(servers, "foo", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") +) +def empty_host(self, tail=20, timeout=20): + """Check that server returns an error when LDAP server + host value is empty. + """ + node = current().context.node + message = "DB::Exception: Empty 'host' entry" + + servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + }] + + with Given("I prepare the error log by writting empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with ldap_servers(servers): + with Then("server shall fail to merge the new config"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") +) +def missing_host(self, tail=20, timeout=20): + """Check that server returns an error when LDAP server + host is missing. + """ + node = current().context.node + message = "DB::Exception: Missing 'host' entry" + + servers = {"foo": {"port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + }] + + with Given("I prepare the error log by writting empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with ldap_servers(servers): + with Then("server shall fail to merge the new config"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), +) +def invalid_port(self): + """Check that server returns an error when LDAP server + port is not valid. + """ + servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server." + }] + login(servers, "openldap1", *users) + + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0") +) +def invalid_auth_dn_prefix(self): + """Check that server returns an error when LDAP server definition + has invalid auth_dn_prefix. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "foo=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + }] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0") +) +def invalid_auth_dn_suffix(self): + """Check that server returns an error when LDAP server definition + has invalid auth_dn_suffix. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",foo=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + }] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0") +) +def invalid_enable_tls_value(self): + """Check that server returns an error when enable_tls + option has invalid value. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap1' is not configured" + }] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0") +) +def invalid_tls_require_cert_value(self): + """Check that server returns an error when tls_require_cert + option has invalid value. + """ + servers = {"openldap2": { + "host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "foo", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap2' is not configured" + }] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir("1.0") +) +def empty_ca_cert_dir(self): + """Check that server returns an error when ca_cert_dir is empty. + """ + servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 20, + "message": "DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain" + }] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile("1.0") +) +def empty_ca_cert_file(self): + """Check that server returns an error when ca_cert_file is empty. + """ + servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 20, + "message": "Received from localhost:9000. DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain)" + }] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Value("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0") +) +def auth_dn_value(self): + """Check that server configuration can properly define the `dn` value of the user.""" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + + login(servers, "openldap1", user) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("1.0") +) +def syntax(self): + """Check that server configuration with valid syntax can be loaded. + ```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + + ``` + """ + servers = { + "openldap2": { + "host": "openldap2", + "port": "389", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "enable_tls": "yes", + "tls_minimum_protocol_version": "tls1.2" , + "tls_require_cert": "demand", + "tls_cert_file": "/container/service/slapd/assets/certs/ldap.crt", + "tls_key_file": "/container/service/slapd/assets/certs/ldap.key", + "tls_ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + "tls_ca_cert_dir": "/container/service/slapd/assets/certs/", + "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384" + } + } + with ldap_servers(servers): + pass + +@TestFeature +@Name("server config") +def feature(self, node="clickhouse1"): + """Check LDAP server configuration. + """ + self.context.node = self.context.cluster.node(node) + for scenario in loads(current_module(), Scenario): + scenario() \ No newline at end of file diff --git a/tests/testflows/ldap/external_user_directory/tests/simple.py b/tests/testflows/ldap/external_user_directory/tests/simple.py new file mode 100644 index 00000000000..c48048833c7 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/simple.py @@ -0,0 +1,24 @@ +from testflows.core import * +from testflows.asserts import error + +from ldap.external_user_directory.tests.common import login + +@TestScenario +@Name("simple") +def scenario(self, node="clickhouse1"): + """Check that an LDAP external user directory can be used to authenticate a user. + """ + self.context.node = self.context.cluster.node(node) + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, + ] + login(servers, "openldap1", *users) diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 8520941ed0b..0e9d06cf84a 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -15,6 +15,7 @@ def regression(self, local, clickhouse_binary_path): args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} Feature(test=load("ldap.authentication.regression", "regression"))(**args) + Feature(test=load("ldap.external_user_directory.regression", "regression"))(**args) if main(): regression() From df09bf3cb67ee130bb545693458ad464d44147ad Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 3 Sep 2020 22:50:26 +0300 Subject: [PATCH 040/441] add suppressions for leak sanitizer --- docker/test/stateful/run.sh | 1 + docker/test/stateless/run.sh | 1 + src/TableFunctions/CMakeLists.txt | 4 + src/TableFunctions/ITableFunction.cpp | 3 +- src/TableFunctions/tests/CMakeLists.txt | 2 + .../tests/test_strange_memory_leak.cpp | 113 ++++++++++++++++++ tests/config/lsan_suppressions.txt | 2 + 7 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 src/TableFunctions/tests/CMakeLists.txt create mode 100644 src/TableFunctions/tests/test_strange_memory_leak.cpp create mode 100644 tests/config/lsan_suppressions.txt diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 5be14970914..e7e1fc54a0a 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -35,6 +35,7 @@ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment +echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment service zookeeper start sleep 5 diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 2ff15ca9c6a..bb587ce73c2 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -53,6 +53,7 @@ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment echo "ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment echo "UBSAN_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment echo "LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-10/bin/llvm-symbolizer" >> /etc/environment +echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment service zookeeper start sleep 5 diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 33e900231e5..19748fefaf9 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -6,3 +6,7 @@ list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFuncti add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) + +if(ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index fd5cdf6e219..804a5b232ec 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -20,8 +20,7 @@ StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & ProfileEvents::increment(ProfileEvents::TableFunctionExecute); context.checkAccess(AccessType::CREATE_TEMPORARY_TABLE | StorageFactory::instance().getSourceAccessType(getStorageTypeName())); - bool no_conversion_required = hasStaticStructure() && cached_columns == getActualTableStructure(context); - if (cached_columns.empty() || no_conversion_required) + if (cached_columns.empty() || (hasStaticStructure() && cached_columns == getActualTableStructure(context))) return executeImpl(ast_function, context, table_name, std::move(cached_columns)); auto get_storage = [=, tf = shared_from_this()]() -> StoragePtr diff --git a/src/TableFunctions/tests/CMakeLists.txt b/src/TableFunctions/tests/CMakeLists.txt new file mode 100644 index 00000000000..f2ed25a0753 --- /dev/null +++ b/src/TableFunctions/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (test-strange-memory-leak test_strange_memory_leak.cpp ${SRCS}) +#target_link_libraries (test-strange-memory-leak PRIVATE dbms) diff --git a/src/TableFunctions/tests/test_strange_memory_leak.cpp b/src/TableFunctions/tests/test_strange_memory_leak.cpp new file mode 100644 index 00000000000..a1e8af34616 --- /dev/null +++ b/src/TableFunctions/tests/test_strange_memory_leak.cpp @@ -0,0 +1,113 @@ +/// Demonstrates mysterious memory leak when built using clang version 10.0.0-4ubuntu1~18.04.2 +/// Ne leaks are found by valgrind when built using gcc version 9.2.1 20191102 + +//#include +//#include + +//class ITableFunction : public std::enable_shared_from_this +//{ +//public: +// virtual DB::ColumnsDescription getActualTableStructure() const = 0; +// virtual ~ITableFunction() {} +// +//}; +// +//class TableFunction : public ITableFunction +//{ +// DB::ColumnsDescription getActualTableStructure() const override +// { +// return DB::ColumnsDescription({{"number", DB::DataTypePtr{}}}); +// } +//}; +// +//template +//class TableFunctionTemplate : public ITableFunction +//{ +// DB::ColumnsDescription getActualTableStructure() const override +// { +// return DB::ColumnsDescription({{"number", DB::DataTypePtr{}}}); +// } +//}; + +/// Simplified version of the commented code above (without dependencies on ClickHouse code): + +#include +#include +#include +#include +#include +#include +#include + +using LolPtr = std::shared_ptr; +struct Elem +{ + Elem(std::string n, LolPtr t) : name(std::move(n)), type(std::move(t)) {} + std::string name; + LolPtr type; +}; + +using Container = boost::multi_index_container< + Elem, + boost::multi_index::indexed_by< + boost::multi_index::sequenced<>, + boost::multi_index::ordered_unique>>>; + +struct List : public std::list +{ + List(std::initializer_list init) : std::list(init) {} +}; + +struct Kek +{ + Container container; + Kek(List list) + { + for (auto & elem : list) + add(Elem(std::move(elem.name), std::move(elem.type))); + } + + void add(Elem column) + { + auto insert_it = container.cbegin(); + container.get<0>().insert(insert_it, std::move(column)); + } + +}; + +class ITableFunction : public std::enable_shared_from_this +{ +public: + virtual Kek getActualTableStructure() const = 0; + virtual ~ITableFunction() {} + +}; + +class TableFunction : public ITableFunction +{ + Kek getActualTableStructure() const override + { + return Kek({{"number", LolPtr{}}}); + } +}; + +template +class TableFunctionTemplate : public ITableFunction +{ + Kek getActualTableStructure() const override + { + return Kek({{"number", LolPtr{}}}); + } +}; + +int main() +{ + /// Works fine + const ITableFunction & tf1 = TableFunction{}; + tf1.getActualTableStructure(); + + /// ERROR: LeakSanitizer: detected memory leaks + /// and the same error with valgrind + const ITableFunction & tf2 = TableFunctionTemplate{}; + tf2.getActualTableStructure(); +} diff --git a/tests/config/lsan_suppressions.txt b/tests/config/lsan_suppressions.txt new file mode 100644 index 00000000000..870bbe5e081 --- /dev/null +++ b/tests/config/lsan_suppressions.txt @@ -0,0 +1,2 @@ +# See src/TableFunctions/tests/test-strange-memory-leak.cpp +leak:getActualTableStructure From 3cb73de0595a4ab3104e81293575a7b62f1c3292 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 4 Sep 2020 01:23:27 +0300 Subject: [PATCH 041/441] try fix fast build --- src/TableFunctions/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TableFunctions/tests/CMakeLists.txt b/src/TableFunctions/tests/CMakeLists.txt index f2ed25a0753..bdc6515ecfd 100644 --- a/src/TableFunctions/tests/CMakeLists.txt +++ b/src/TableFunctions/tests/CMakeLists.txt @@ -1,2 +1,3 @@ add_executable (test-strange-memory-leak test_strange_memory_leak.cpp ${SRCS}) #target_link_libraries (test-strange-memory-leak PRIVATE dbms) +target_link_libraries (test-strange-memory-leak PUBLIC boost::headers_only) From c233be29c57a694c3e64fd9182c68da8c1592975 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 4 Sep 2020 16:29:43 +0300 Subject: [PATCH 042/441] fix --- src/Interpreters/InterpreterCreateQuery.cpp | 1 + src/TableFunctions/tests/CMakeLists.txt | 2 +- .../tests/test_strange_memory_leak.cpp | 45 ++++++++++++++----- src/TableFunctions/ya.make | 1 + .../test_distributed_format/test.py | 6 +-- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 20825f35109..d053d8e6870 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -587,6 +587,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.storage || create.is_view || create.is_materialized_view || create.is_live_view || create.is_dictionary) { + assert(create.storage->engine); if (create.temporary && create.storage->engine->name != "Memory") throw Exception( "Temporary tables can only be created with ENGINE = Memory, not " + create.storage->engine->name, diff --git a/src/TableFunctions/tests/CMakeLists.txt b/src/TableFunctions/tests/CMakeLists.txt index bdc6515ecfd..6b70ce0beca 100644 --- a/src/TableFunctions/tests/CMakeLists.txt +++ b/src/TableFunctions/tests/CMakeLists.txt @@ -1,3 +1,3 @@ add_executable (test-strange-memory-leak test_strange_memory_leak.cpp ${SRCS}) #target_link_libraries (test-strange-memory-leak PRIVATE dbms) -target_link_libraries (test-strange-memory-leak PUBLIC boost::headers_only) +#target_link_libraries (test-strange-memory-leak PUBLIC boost::headers_only) diff --git a/src/TableFunctions/tests/test_strange_memory_leak.cpp b/src/TableFunctions/tests/test_strange_memory_leak.cpp index a1e8af34616..dcf6c779429 100644 --- a/src/TableFunctions/tests/test_strange_memory_leak.cpp +++ b/src/TableFunctions/tests/test_strange_memory_leak.cpp @@ -1,5 +1,5 @@ /// Demonstrates mysterious memory leak when built using clang version 10.0.0-4ubuntu1~18.04.2 -/// Ne leaks are found by valgrind when built using gcc version 9.2.1 20191102 +/// No leaks are found by valgrind when built using gcc version 9.2.1 20191102 //#include //#include @@ -34,10 +34,23 @@ #include #include #include -#include -#include -#include -#include +#include +#include +//#include +//#include +//#include +//#include + +struct Print +{ + std::string data = "lol"; + Print(const Print &) { std::cout << "copy" << std::endl; } + Print(Print &&) { std::cout << "move" << std::endl; } + Print & operator = (const Print &) { std::cout << "copy=" << std::endl; return *this; } + Print & operator = (Print &&) { std::cout << "move=" << std::endl; return *this; } + Print() { std::cout << "ctor" << std::endl; } + ~Print() { std::cout << "dtor" << std::endl; } +}; using LolPtr = std::shared_ptr; struct Elem @@ -45,13 +58,16 @@ struct Elem Elem(std::string n, LolPtr t) : name(std::move(n)), type(std::move(t)) {} std::string name; LolPtr type; + Print print; }; -using Container = boost::multi_index_container< - Elem, - boost::multi_index::indexed_by< - boost::multi_index::sequenced<>, - boost::multi_index::ordered_unique>>>; +//using Container = boost::multi_index_container< +// Elem, +// boost::multi_index::indexed_by< +// boost::multi_index::sequenced<>, +// boost::multi_index::ordered_unique>>>; +/// Simplified version: +using Container = std::map; struct List : public std::list { @@ -70,7 +86,8 @@ struct Kek void add(Elem column) { auto insert_it = container.cbegin(); - container.get<0>().insert(insert_it, std::move(column)); + //container.get<0>().insert(insert_it, std::move(column)); + container.insert(insert_it, {column.name, column}); } }; @@ -102,12 +119,18 @@ class TableFunctionTemplate : public ITableFunction int main() { + std::cout << "0" << std::endl; + /// Works fine const ITableFunction & tf1 = TableFunction{}; tf1.getActualTableStructure(); + std::cout << "1" << std::endl; + /// ERROR: LeakSanitizer: detected memory leaks /// and the same error with valgrind const ITableFunction & tf2 = TableFunctionTemplate{}; tf2.getActualTableStructure(); + + std::cout << "2" << std::endl; } diff --git a/src/TableFunctions/ya.make b/src/TableFunctions/ya.make index e87c96073bd..36218bee1e0 100644 --- a/src/TableFunctions/ya.make +++ b/src/TableFunctions/ya.make @@ -23,6 +23,7 @@ SRCS( TableFunctionValues.cpp TableFunctionView.cpp TableFunctionZeros.cpp + tests/test_strange_memory_leak.cpp ) diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py index 291db89ae4c..08d2f0087a4 100644 --- a/tests/integration/test_distributed_format/test.py +++ b/tests/integration/test_distributed_format/test.py @@ -37,7 +37,7 @@ def test_single_file(started_cluster, cluster): assert out == '1\ta\n2\tbb\n3\tccc\n' - query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin');" \ + query = "create table t (x UInt64, s String) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_1/shard1_replica1/1.bin');" \ "select * from t" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) @@ -57,7 +57,7 @@ def test_two_files(started_cluster, cluster): assert out == '0\t_\n1\ta\n2\tbb\n3\tccc\n' - query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin');" \ + query = "create table t (x UInt64, s String) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_2/shard1_replica1/{1,2,3,4}.bin');" \ "select * from t order by x" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) @@ -76,7 +76,7 @@ def test_single_file_old(started_cluster, cluster): assert out == '1\ta\n2\tbb\n3\tccc\n' - query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin');" \ + query = "create table t (x UInt64, s String) engine = File('Distributed', '/var/lib/clickhouse/data/test/distr_3/default@not_existing:9000/1.bin');" \ "select * from t" out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query]) From 98963deed33812d66bd4159c6ab1b98377fbee81 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 4 Sep 2020 19:31:37 +0300 Subject: [PATCH 043/441] fix --- src/Interpreters/InterpreterCreateQuery.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d053d8e6870..0e7c2e5a48f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -587,8 +587,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.storage || create.is_view || create.is_materialized_view || create.is_live_view || create.is_dictionary) { - assert(create.storage->engine); - if (create.temporary && create.storage->engine->name != "Memory") + if (create.temporary && create.storage && create.storage->engine && create.storage->engine->name != "Memory") throw Exception( "Temporary tables can only be created with ENGINE = Memory, not " + create.storage->engine->name, ErrorCodes::INCORRECT_QUERY); From 88e5476dbe6c9491427b7e2f4088fe81ac7d23c4 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 5 Sep 2020 00:40:31 +0300 Subject: [PATCH 044/441] Docs for the cast_keep_nullable setting (English). --- docs/en/operations/settings/settings.md | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4995c04f712..9bbdcacd1ab 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1899,4 +1899,53 @@ Possible values: Default value: `120`. +## cast_keep_nullable {#cast_keep_nullable} + +Enables or disables converstion to nullable types for [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) operator. + +If set, `CAST(something_nullable AS Type)` returns `Nullable(Type)`. + +Possible values: + +- 0 — Nullable type converstion disabled. +- 1 — Nullable type converstion enabled. + +Default value: `0`. + +**Examples** + +Query with nullable type converstion disabled: + +```sql +SET cast_keep_nullable = 0; +SELECT CAST(toNullable(toInt32(0)) AS Int32) as x, toTypeName(x); +``` + +Result: + +``` +┌─x─┬─toTypeName(CAST(toNullable(toInt32(0)), 'Int32'))─┐ +│ 0 │ Int32 │ +└───┴───────────────────────────────────────────────────┘ +``` + +Query with nullable type converstion enabled: + +```sql +SET cast_keep_nullable = 1; +SELECT CAST(toNullable(toInt32(0)) AS Int32) as x, toTypeName(x); +``` + +Result: + +``` +┌─x─┬─toTypeName(CAST(toNullable(toInt32(0)), 'Int32'))─┐ +│ 0 │ Nullable(Int32) │ +└───┴───────────────────────────────────────────────────┘ +``` + +**See Also** + +- [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) operator + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From a76139bc223ec952c5c852b0cecd5dd285b20c9d Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 5 Sep 2020 01:00:07 +0300 Subject: [PATCH 045/441] Minor fixes. --- docs/en/operations/settings/settings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 9bbdcacd1ab..dc93d31dea4 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1923,7 +1923,7 @@ SELECT CAST(toNullable(toInt32(0)) AS Int32) as x, toTypeName(x); Result: -``` +```text ┌─x─┬─toTypeName(CAST(toNullable(toInt32(0)), 'Int32'))─┐ │ 0 │ Int32 │ └───┴───────────────────────────────────────────────────┘ @@ -1938,7 +1938,7 @@ SELECT CAST(toNullable(toInt32(0)) AS Int32) as x, toTypeName(x); Result: -``` +```text ┌─x─┬─toTypeName(CAST(toNullable(toInt32(0)), 'Int32'))─┐ │ 0 │ Nullable(Int32) │ └───┴───────────────────────────────────────────────────┘ From c5df4647e6f5571b55e72f680e4ba8eb214e5c96 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 7 Sep 2020 14:30:12 +0300 Subject: [PATCH 046/441] add workaround for memory leak --- src/TableFunctions/CMakeLists.txt | 3 - src/TableFunctions/TableFunctionNumbers.cpp | 3 +- src/TableFunctions/TableFunctionZeros.cpp | 3 +- src/TableFunctions/tests/CMakeLists.txt | 3 - .../tests/test_strange_memory_leak.cpp | 136 ------------------ src/TableFunctions/ya.make | 1 - tests/config/lsan_suppressions.txt | 4 +- 7 files changed, 6 insertions(+), 147 deletions(-) delete mode 100644 src/TableFunctions/tests/CMakeLists.txt delete mode 100644 src/TableFunctions/tests/test_strange_memory_leak.cpp diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 19748fefaf9..8e9eedadf53 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -7,6 +7,3 @@ list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFuncti add_library(clickhouse_table_functions ${clickhouse_table_functions_sources}) target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) -if(ENABLE_TESTS) - add_subdirectory(tests) -endif() diff --git a/src/TableFunctions/TableFunctionNumbers.cpp b/src/TableFunctions/TableFunctionNumbers.cpp index 984b26d5de4..4658165735a 100644 --- a/src/TableFunctions/TableFunctionNumbers.cpp +++ b/src/TableFunctions/TableFunctionNumbers.cpp @@ -23,7 +23,8 @@ namespace ErrorCodes template ColumnsDescription TableFunctionNumbers::getActualTableStructure(const Context & /*context*/) const { - return ColumnsDescription({{"number", std::make_shared()}}); + /// NOTE: https://bugs.llvm.org/show_bug.cgi?id=47418 + return ColumnsDescription{{{"number", std::make_shared()}}}; } template diff --git a/src/TableFunctions/TableFunctionZeros.cpp b/src/TableFunctions/TableFunctionZeros.cpp index 22bf47f5e9f..9b0c6c6e78b 100644 --- a/src/TableFunctions/TableFunctionZeros.cpp +++ b/src/TableFunctions/TableFunctionZeros.cpp @@ -22,7 +22,8 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; template ColumnsDescription TableFunctionZeros::getActualTableStructure(const Context & /*context*/) const { - return ColumnsDescription({{"zero", std::make_shared()}}); + /// NOTE: https://bugs.llvm.org/show_bug.cgi?id=47418 + return ColumnsDescription{{{"zero", std::make_shared()}}}; } template diff --git a/src/TableFunctions/tests/CMakeLists.txt b/src/TableFunctions/tests/CMakeLists.txt deleted file mode 100644 index 6b70ce0beca..00000000000 --- a/src/TableFunctions/tests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_executable (test-strange-memory-leak test_strange_memory_leak.cpp ${SRCS}) -#target_link_libraries (test-strange-memory-leak PRIVATE dbms) -#target_link_libraries (test-strange-memory-leak PUBLIC boost::headers_only) diff --git a/src/TableFunctions/tests/test_strange_memory_leak.cpp b/src/TableFunctions/tests/test_strange_memory_leak.cpp deleted file mode 100644 index dcf6c779429..00000000000 --- a/src/TableFunctions/tests/test_strange_memory_leak.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/// Demonstrates mysterious memory leak when built using clang version 10.0.0-4ubuntu1~18.04.2 -/// No leaks are found by valgrind when built using gcc version 9.2.1 20191102 - -//#include -//#include - -//class ITableFunction : public std::enable_shared_from_this -//{ -//public: -// virtual DB::ColumnsDescription getActualTableStructure() const = 0; -// virtual ~ITableFunction() {} -// -//}; -// -//class TableFunction : public ITableFunction -//{ -// DB::ColumnsDescription getActualTableStructure() const override -// { -// return DB::ColumnsDescription({{"number", DB::DataTypePtr{}}}); -// } -//}; -// -//template -//class TableFunctionTemplate : public ITableFunction -//{ -// DB::ColumnsDescription getActualTableStructure() const override -// { -// return DB::ColumnsDescription({{"number", DB::DataTypePtr{}}}); -// } -//}; - -/// Simplified version of the commented code above (without dependencies on ClickHouse code): - -#include -#include -#include -#include -#include -//#include -//#include -//#include -//#include - -struct Print -{ - std::string data = "lol"; - Print(const Print &) { std::cout << "copy" << std::endl; } - Print(Print &&) { std::cout << "move" << std::endl; } - Print & operator = (const Print &) { std::cout << "copy=" << std::endl; return *this; } - Print & operator = (Print &&) { std::cout << "move=" << std::endl; return *this; } - Print() { std::cout << "ctor" << std::endl; } - ~Print() { std::cout << "dtor" << std::endl; } -}; - -using LolPtr = std::shared_ptr; -struct Elem -{ - Elem(std::string n, LolPtr t) : name(std::move(n)), type(std::move(t)) {} - std::string name; - LolPtr type; - Print print; -}; - -//using Container = boost::multi_index_container< -// Elem, -// boost::multi_index::indexed_by< -// boost::multi_index::sequenced<>, -// boost::multi_index::ordered_unique>>>; -/// Simplified version: -using Container = std::map; - -struct List : public std::list -{ - List(std::initializer_list init) : std::list(init) {} -}; - -struct Kek -{ - Container container; - Kek(List list) - { - for (auto & elem : list) - add(Elem(std::move(elem.name), std::move(elem.type))); - } - - void add(Elem column) - { - auto insert_it = container.cbegin(); - //container.get<0>().insert(insert_it, std::move(column)); - container.insert(insert_it, {column.name, column}); - } - -}; - -class ITableFunction : public std::enable_shared_from_this -{ -public: - virtual Kek getActualTableStructure() const = 0; - virtual ~ITableFunction() {} - -}; - -class TableFunction : public ITableFunction -{ - Kek getActualTableStructure() const override - { - return Kek({{"number", LolPtr{}}}); - } -}; - -template -class TableFunctionTemplate : public ITableFunction -{ - Kek getActualTableStructure() const override - { - return Kek({{"number", LolPtr{}}}); - } -}; - -int main() -{ - std::cout << "0" << std::endl; - - /// Works fine - const ITableFunction & tf1 = TableFunction{}; - tf1.getActualTableStructure(); - - std::cout << "1" << std::endl; - - /// ERROR: LeakSanitizer: detected memory leaks - /// and the same error with valgrind - const ITableFunction & tf2 = TableFunctionTemplate{}; - tf2.getActualTableStructure(); - - std::cout << "2" << std::endl; -} diff --git a/src/TableFunctions/ya.make b/src/TableFunctions/ya.make index 36218bee1e0..e87c96073bd 100644 --- a/src/TableFunctions/ya.make +++ b/src/TableFunctions/ya.make @@ -23,7 +23,6 @@ SRCS( TableFunctionValues.cpp TableFunctionView.cpp TableFunctionZeros.cpp - tests/test_strange_memory_leak.cpp ) diff --git a/tests/config/lsan_suppressions.txt b/tests/config/lsan_suppressions.txt index 870bbe5e081..39eb40560d7 100644 --- a/tests/config/lsan_suppressions.txt +++ b/tests/config/lsan_suppressions.txt @@ -1,2 +1,2 @@ -# See src/TableFunctions/tests/test-strange-memory-leak.cpp -leak:getActualTableStructure +# See https://bugs.llvm.org/show_bug.cgi?id=47418 +# leak:getActualTableStructure From d6c84291c95ffc0f5fd96c206747033ab6266522 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 8 Sep 2020 16:28:52 +0300 Subject: [PATCH 047/441] Fixed data race caused by EVP_CIPHER_fetch() --- src/Functions/FunctionsAES.cpp | 26 +++++--------------------- src/Functions/FunctionsAES.h | 14 +++++--------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 009226c035a..0d459b6095f 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -7,18 +7,6 @@ #include -namespace -{ -void CipherDeleter(const EVP_CIPHER * cipher [[maybe_unused]]) -{ -#if OPENSSL_VERSION_NUMBER >= 0x03 << 28 -// Used to free EVP_CIPHER poniter obtained with EVP_CIPHER_fetch, -// available only since OpenSSL ver 3.0.0. - EVP_CIPHER_free(const_cast(cipher)); -#endif -} -} - namespace DB { namespace ErrorCodes @@ -47,7 +35,7 @@ StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const return StringRef(folded_key.data(), cipher_key_size); } -CipherPtr getCipherByName(const StringRef & cipher_name) +const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) { const auto *evp_cipher = EVP_get_cipherbyname(cipher_name.data); if (evp_cipher == nullptr) @@ -61,14 +49,10 @@ CipherPtr getCipherByName(const StringRef & cipher_name) evp_cipher = EVP_aes_256_cfb128(); } -#if OPENSSL_VERSION_NUMBER < 0x03 << 28 - return CipherPtr{evp_cipher, CipherDeleter}; -#else - // HACK: To speed up context initialization with EVP_EncryptInit_ex (which is called at least once per row) - // Apparently cipher from EVP_get_cipherbyname may require additional initialization of context, - // while cipher from EVP_CIPHER_fetch causes less operations => faster context initialization. - return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &CipherDeleter}; -#endif + // NOTE: cipher obtained not via EVP_CIPHER_fetch() would cause extra work on each context reset + // with EVP_CIPHER_CTX_reset() or EVP_EncryptInit_ex(), but using EVP_CIPHER_fetch() + // causes data race, so we stick to the slower but safer alternative here. + return evp_cipher; } } diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 25b41a067a7..8465c35a517 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -36,9 +36,7 @@ namespace OpenSSLDetails [[noreturn]] void onError(std::string error_message); StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); -using CipherDeleterType = void (*) (const EVP_CIPHER *cipher); -using CipherPtr = std::unique_ptr; -CipherPtr getCipherByName(const StringRef & name); +const EVP_CIPHER * getCipherByName(const StringRef & name); enum class CompatibilityMode { @@ -189,10 +187,9 @@ private: if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - auto cipher = getCipherByName(mode); - if (cipher == nullptr) + auto evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const EVP_CIPHER * evp_cipher = cipher.get(); const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); @@ -438,10 +435,9 @@ private: if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - auto cipher = getCipherByName(mode); - if (cipher == nullptr) + auto evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const EVP_CIPHER * evp_cipher = cipher.get(); OpenSSLDetails::validateCipherMode(evp_cipher); From f727f7bc99fb1ae606fe5a5dc7bc6010e1802ab0 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 9 Sep 2020 16:14:02 +0200 Subject: [PATCH 048/441] * Updating config.xml files to remove users_config and access_control_path parameter definition as these are now defined inside the user_directories section and cause a test failure due to https://github.com/ClickHouse/ClickHouse/issues/14511. * Removing the usage of `EXISTS` clauses used in tests outside of clean ups. --- .../ldap/authentication/configs/clickhouse/config.xml | 6 ------ .../external_user_directory/configs/clickhouse/config.xml | 6 ------ .../testflows/ldap/external_user_directory/tests/roles.py | 8 ++++---- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/testflows/ldap/authentication/configs/clickhouse/config.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml index 80c3150d326..e28a0c8e255 100644 --- a/tests/testflows/ldap/authentication/configs/clickhouse/config.xml +++ b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml @@ -117,9 +117,6 @@ /var/lib/clickhouse/user_files/ - - /var/lib/clickhouse/access/ - @@ -132,9 +129,6 @@ - - users.xml - default diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml index 80c3150d326..e28a0c8e255 100644 --- a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml @@ -117,9 +117,6 @@ /var/lib/clickhouse/user_files/ - - /var/lib/clickhouse/access/ - @@ -132,9 +129,6 @@ - - users.xml - default diff --git a/tests/testflows/ldap/external_user_directory/tests/roles.py b/tests/testflows/ldap/external_user_directory/tests/roles.py index d621993078b..8a6c6f465d1 100644 --- a/tests/testflows/ldap/external_user_directory/tests/roles.py +++ b/tests/testflows/ldap/external_user_directory/tests/roles.py @@ -166,7 +166,7 @@ def remove_role(self, server, timeout=20): settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) with And("I remove one of the roles"): - node.query(f"DROP ROLE IF EXISTS {roles[1]}") + node.query(f"DROP ROLE {roles[1]}") with And(f"I try to login using cached LDAP user"): node.query(f"SELECT 1", @@ -212,7 +212,7 @@ def remove_privilege_by_removing_role(self, server, timeout=20): settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) with And("I remove the role that grants the privilege"): - node.query(f"DROP ROLE IF EXISTS {roles[0]}") + node.query(f"DROP ROLE {roles[0]}") with And(f"I try to relogin and expect that cached LDAP user can login " "but does not have privilege that was provided by the removed role"): @@ -254,7 +254,7 @@ def readd_privilege_by_readding_role(self, server, timeout=20): settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) with And("I remove the role that grants the privilege"): - node.query(f"DROP ROLE IF EXISTS {roles[0]}") + node.query(f"DROP ROLE {roles[0]}") message = "DB::Exception: {user}: Not enough privileges." exitcode = 241 @@ -274,7 +274,7 @@ def readd_privilege_by_readding_role(self, server, timeout=20): exitcode=exitcode, message=message.format(user=users[1]["username"])) with When("I re-add the role"): - node.query(f"CREATE ROLE IF NOT EXISTS {roles[0]}") + node.query(f"CREATE ROLE {roles[0]}") with And(f"I grant select privilege to the re-added role"): node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") From ef7ca3788dadb352153b2f0aa35e48e61be7f45f Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 9 Sep 2020 19:16:41 +0300 Subject: [PATCH 049/441] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 698b38460e4..a9267c3ed5c 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -140,3 +140,4 @@ 01456_ast_optimizations_over_distributed 01460_DistributedFilesToInsert 01474_bad_global_join +01457_create_as_table_function_structure From 8129169ead0e2ac3f7f7eb836732b3212dc317dc Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 10 Sep 2020 22:52:42 +0300 Subject: [PATCH 050/441] Fixed Yandex-specific builds :( --- src/Functions/FunctionsAES.h | 4 +++- src/Functions/aes_decrypt_mysql.cpp | 4 +++- src/Functions/aes_encrypt_mysql.cpp | 4 +++- src/Functions/decrypt.cpp | 4 +++- src/Functions/encrypt.cpp | 4 +++- src/Functions/registerFunctions.cpp | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 8465c35a517..fc9b9ef6f40 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -1,6 +1,8 @@ #pragma once -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL #include diff --git a/src/Functions/aes_decrypt_mysql.cpp b/src/Functions/aes_decrypt_mysql.cpp index 764fcf06c1a..d5d7a6660e4 100644 --- a/src/Functions/aes_decrypt_mysql.cpp +++ b/src/Functions/aes_decrypt_mysql.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp index 1d84824d9d6..bdf7d8d2cce 100644 --- a/src/Functions/aes_encrypt_mysql.cpp +++ b/src/Functions/aes_encrypt_mysql.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/decrypt.cpp b/src/Functions/decrypt.cpp index 1cbda0dba99..a9090a31f0e 100644 --- a/src/Functions/decrypt.cpp +++ b/src/Functions/decrypt.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/encrypt.cpp b/src/Functions/encrypt.cpp index 807443a2fb8..ba8ffa1bc59 100644 --- a/src/Functions/encrypt.cpp +++ b/src/Functions/encrypt.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index b7fb3cecd66..e88a6704b56 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #include From bb1d126ce53794d814f5d7d9dfce2c62b51dd13e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 9 Sep 2020 23:45:07 +0200 Subject: [PATCH 051/441] Updating TestFlows to 1.6.45 to fix stability issues with multi line commands. --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 6b4ec12b80c..219873e7be6 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.42 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.45 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce From fe35de5fec942cd3aff88a0d819d8aec58a1d6af Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 11 Sep 2020 22:41:22 +0200 Subject: [PATCH 052/441] Updating TestFlows to 1.6.46 --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 898552ade56..ecbb43a88ec 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.42 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.46 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce From 30b1831752209d1aabcf4ff80dc8d1c2bd919b27 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 14 Sep 2020 18:15:07 +0300 Subject: [PATCH 053/441] Moved default values for query_masking rules for encrypt/decrypt to config.xml --- debian/clickhouse-server.install | 1 - programs/server/CMakeLists.txt | 2 -- programs/server/config.xml | 14 +++++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install index 57f858407af..b1475fdf162 100644 --- a/debian/clickhouse-server.install +++ b/debian/clickhouse-server.install @@ -2,6 +2,5 @@ usr/bin/clickhouse-server usr/bin/clickhouse-copier usr/bin/clickhouse-report etc/clickhouse-server/config.xml -etc/clickhouse-server/config.d/*.xml etc/clickhouse-server/users.xml etc/systemd/system/clickhouse-server.service diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index adaddcf1762..5500a4680b7 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -29,8 +29,6 @@ set (CLICKHOUSE_SERVER_LINK clickhouse_program_add(server) install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) -install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) -install(FILES config.d/query_masking_rules.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server/config.d COMPONENT clickhouse) # TODO We actually need this on Mac, FreeBSD. if (OS_LINUX) diff --git a/programs/server/config.xml b/programs/server/config.xml index af01e880dc2..fc41e323e43 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -670,18 +670,22 @@ --> /var/lib/clickhouse/format_schemas/ - - hide SSN - \b\d{3}-\d{2}-\d{4}\b - 000-00-0000 + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) - --> From 9d6247bb7ed0064c08ba7b5ee8dda2a0335839de Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 15 Sep 2020 14:38:22 +0300 Subject: [PATCH 058/441] Links fixed --- docs/ru/interfaces/formats.md | 4 ++-- docs/ru/operations/settings/settings.md | 2 +- docs/ru/sql-reference/functions/bitmap-functions.md | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 04bca115974..dd68f7eb646 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1050,13 +1050,13 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ Для обмена данными с экосистемой Hadoop можно использовать движки таблиц [HDFS](../engines/table-engines/integrations/hdfs.md). -## Arrow {data-format-arrow} +## Arrow {#data-format-arrow} [Apache Arrow](https://arrow.apache.org/) поставляется с двумя встроенными поколоночнами форматами хранения. ClickHouse поддерживает операции чтения и записи для этих форматов. `Arrow` — это Apache Arrow's "file mode" формат. Он предназначен для произвольного доступа в памяти. -## ArrowStream {data-format-arrow-stream} +## ArrowStream {#data-format-arrow-stream} `ArrowStream` — это Apache Arrow's "stream mode" формат. Он предназначен для обработки потоков в памяти. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 70e20ef954d..0667637c4ad 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -484,7 +484,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( См. также: -- [JOIN strictness](../../sql-reference/statements/select/join.md#select-join-strictness) +- [JOIN strictness](../../sql-reference/statements/select/join.md#join-settings) ## max\_block\_size {#setting-max_block_size} diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index c91725c7a39..ef59a46d59d 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -61,8 +61,8 @@ bitmapSubsetLimit(bitmap, range_start, cardinality_limit) **Параметры** - `bitmap` – Битмап. [Bitmap object](#bitmap_functions-bitmapbuild). -- `range_start` – Начальная точка подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md). -- `cardinality_limit` – Верхний предел подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md). +- `range_start` – Начальная точка подмножества. [UInt32](../../sql-reference/data-types/int-uint.md). +- `cardinality_limit` – Верхний предел подмножества. [UInt32](../../sql-reference/data-types/int-uint.md). **Возвращаемое значение** @@ -97,7 +97,7 @@ bitmapContains(haystack, needle) **Параметры** - `haystack` – [объект Bitmap](#bitmap_functions-bitmapbuild), в котором функция ищет значение. -- `needle` – значение, которое функция ищет. Тип — [UInt32](../../sql-reference/functions/bitmap-functions.md). +- `needle` – значение, которое функция ищет. Тип — [UInt32](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** From 320add704792de80a10fe0e148d11b53ada6c10a Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Mon, 14 Sep 2020 20:19:13 -0400 Subject: [PATCH 059/441] Adding source for the SRS. --- .../requirements/requirements.md | 1926 +++++++++++++++++ 1 file changed, 1926 insertions(+) create mode 100644 tests/testflows/aes_encryption/requirements/requirements.md diff --git a/tests/testflows/aes_encryption/requirements/requirements.md b/tests/testflows/aes_encryption/requirements/requirements.md new file mode 100644 index 00000000000..da61c85893e --- /dev/null +++ b/tests/testflows/aes_encryption/requirements/requirements.md @@ -0,0 +1,1926 @@ +# SRS-008 ClickHouse AES Encryption Functions +# Software Requirements Specification + +## Table of Contents +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS008.AES.Functions](#rqsrs008aesfunctions) + * 4.1.2 [RQ.SRS008.AES.Functions.Compatability.MySQL](#rqsrs008aesfunctionscompatabilitymysql) + * 4.1.3 [RQ.SRS008.AES.Functions.Compatability.Dictionaries](#rqsrs008aesfunctionscompatabilitydictionaries) + * 4.1.4 [RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL](#rqsrs008aesfunctionscompatabilityenginedatabasemysql) + * 4.1.5 [RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL](#rqsrs008aesfunctionscompatabilityenginetablemysql) + * 4.1.6 [RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL](#rqsrs008aesfunctionscompatabilitytablefunctionmysql) + * 4.1.7 [RQ.SRS008.AES.Functions.DifferentModes](#rqsrs008aesfunctionsdifferentmodes) + * 4.1.8 [RQ.SRS008.AES.Functions.DataFromMultipleSources](#rqsrs008aesfunctionsdatafrommultiplesources) + * 4.1.9 [RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues](#rqsrs008aesfunctionssuppressoutputofsensitivevalues) + * 4.1.10 [RQ.SRS008.AES.Functions.InvalidParameters](#rqsrs008aesfunctionsinvalidparameters) + * 4.1.11 [RQ.SRS008.AES.Functions.MismatchedKey](#rqsrs008aesfunctionsmismatchedkey) + * 4.1.12 [RQ.SRS008.AES.Functions.Check.Performance](#rqsrs008aesfunctionscheckperformance) + * 4.1.13 [RQ.SRS008.AES.Function.Check.Performance.BestCase](#rqsrs008aesfunctioncheckperformancebestcase) + * 4.1.14 [RQ.SRS008.AES.Function.Check.Performance.WorstCase](#rqsrs008aesfunctioncheckperformanceworstcase) + * 4.1.15 [RQ.SRS008.AES.Functions.Check.Compression](#rqsrs008aesfunctionscheckcompression) + * 4.1.16 [RQ.SRS008.AES.Functions.Check.Compression.LowCardinality](#rqsrs008aesfunctionscheckcompressionlowcardinality) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS008.AES.Encrypt.Function](#rqsrs008aesencryptfunction) + * 4.2.2 [RQ.SRS008.AES.Encrypt.Function.Syntax](#rqsrs008aesencryptfunctionsyntax) + * 4.2.3 [RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors](#rqsrs008aesencryptfunctionnisttestvectors) + * 4.2.4 [RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText](#rqsrs008aesencryptfunctionparametersplaintext) + * 4.2.5 [RQ.SRS008.AES.Encrypt.Function.Parameters.Key](#rqsrs008aesencryptfunctionparameterskey) + * 4.2.6 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode](#rqsrs008aesencryptfunctionparametersmode) + * 4.2.7 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesencryptfunctionparametersmodevaluesformat) + * 4.2.8 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesencryptfunctionparametersmodevalueinvalid) + * 4.2.9 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.10 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.11 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.12 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.13 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.14 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.15 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.16 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.17 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.18 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.19 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.20 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.21 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.22 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.23 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.24 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.25 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.26 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.27 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.28 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.29 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.30 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.31 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.32 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.33 [RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesencryptfunctionparametersinitializationvector) + * 4.2.34 [RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesencryptfunctionparametersadditionalauthenticateddata) + * 4.2.35 [RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesencryptfunctionparametersreturnvalue) + * 4.2.36 [RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesencryptfunctionkeylengthinvalidlengtherror) + * 4.2.37 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesencryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.38 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesencryptfunctioninitializationvectornotvalidformode) + * 4.2.39 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesencryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.40 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesencryptfunctionadditionalauthenticationdatalength) + * 4.2.41 [RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.42 [RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.43 [RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.44 [RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.45 [RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.46 [RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.47 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.48 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.49 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.50 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.51 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.52 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.53 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.54 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.55 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.56 [RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.57 [RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.58 [RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.59 [RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.60 [RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.61 [RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.62 [RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.63 [RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.64 [RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.2.65 [RQ.SRS008.AES.Decrypt.Function](#rqsrs008aesdecryptfunction) + * 4.2.66 [RQ.SRS008.AES.Decrypt.Function.Syntax](#rqsrs008aesdecryptfunctionsyntax) + * 4.2.67 [RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText](#rqsrs008aesdecryptfunctionparametersciphertext) + * 4.2.68 [RQ.SRS008.AES.Decrypt.Function.Parameters.Key](#rqsrs008aesdecryptfunctionparameterskey) + * 4.2.69 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode](#rqsrs008aesdecryptfunctionparametersmode) + * 4.2.70 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesdecryptfunctionparametersmodevaluesformat) + * 4.2.71 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesdecryptfunctionparametersmodevalueinvalid) + * 4.2.72 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.73 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.74 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.75 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.76 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.77 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.78 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.79 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.80 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.81 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.82 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.83 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.84 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.85 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.86 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.87 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.88 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.89 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.90 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.91 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.92 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.93 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.94 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.95 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.96 [RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesdecryptfunctionparametersinitializationvector) + * 4.2.97 [RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesdecryptfunctionparametersadditionalauthenticateddata) + * 4.2.98 [RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesdecryptfunctionparametersreturnvalue) + * 4.2.99 [RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesdecryptfunctionkeylengthinvalidlengtherror) + * 4.2.100 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesdecryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.101 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesdecryptfunctioninitializationvectornotvalidformode) + * 4.2.102 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesdecryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.103 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesdecryptfunctionadditionalauthenticationdatalength) + * 4.2.104 [RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.105 [RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.106 [RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.107 [RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.108 [RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.109 [RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.110 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.111 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.112 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.113 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.114 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.115 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.116 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.117 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.118 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.119 [RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.120 [RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.121 [RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.122 [RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.123 [RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.124 [RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.125 [RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.126 [RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.127 [RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.3 [MySQL Specific Functions](#mysql-specific-functions) + * 4.3.1 [RQ.SRS008.AES.MySQL.Encrypt.Function](#rqsrs008aesmysqlencryptfunction) + * 4.3.2 [RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax](#rqsrs008aesmysqlencryptfunctionsyntax) + * 4.3.3 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText](#rqsrs008aesmysqlencryptfunctionparametersplaintext) + * 4.3.4 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key](#rqsrs008aesmysqlencryptfunctionparameterskey) + * 4.3.5 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode](#rqsrs008aesmysqlencryptfunctionparametersmode) + * 4.3.6 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesformat) + * 4.3.7 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqlencryptfunctionparametersmodevalueinvalid) + * 4.3.8 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.9 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.10 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.11 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.12 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.13 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.14 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.15 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.16 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.17 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.18 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.19 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.20 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.21 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.22 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.23 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.24 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.25 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.26 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.27 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.28 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.29 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.30 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.31 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.32 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqlencryptfunctionparametersinitializationvector) + * 4.3.33 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqlencryptfunctionparametersreturnvalue) + * 4.3.34 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqlencryptfunctionkeylengthtooshorterror) + * 4.3.35 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqlencryptfunctionkeylengthtoolong) + * 4.3.36 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.37 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtoolong) + * 4.3.38 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqlencryptfunctioninitializationvectornotvalidformode) + * 4.3.39 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.40 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.41 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.42 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.43 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.44 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.45 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.46 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.47 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.48 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.49 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.50 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.51 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.52 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.53 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.54 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.55 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.56 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.3.57 [RQ.SRS008.AES.MySQL.Decrypt.Function](#rqsrs008aesmysqldecryptfunction) + * 4.3.58 [RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax](#rqsrs008aesmysqldecryptfunctionsyntax) + * 4.3.59 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText](#rqsrs008aesmysqldecryptfunctionparametersciphertext) + * 4.3.60 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key](#rqsrs008aesmysqldecryptfunctionparameterskey) + * 4.3.61 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode](#rqsrs008aesmysqldecryptfunctionparametersmode) + * 4.3.62 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesformat) + * 4.3.63 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqldecryptfunctionparametersmodevalueinvalid) + * 4.3.64 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.65 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.66 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.67 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.68 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.69 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.70 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.71 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.72 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.73 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.74 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.75 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.76 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.77 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.78 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.79 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.80 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.81 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.82 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.83 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.84 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.85 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.86 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.87 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.88 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqldecryptfunctionparametersinitializationvector) + * 4.3.89 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqldecryptfunctionparametersreturnvalue) + * 4.3.90 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqldecryptfunctionkeylengthtooshorterror) + * 4.3.91 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqldecryptfunctionkeylengthtoolong) + * 4.3.92 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.93 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtoolong) + * 4.3.94 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqldecryptfunctioninitializationvectornotvalidformode) + * 4.3.95 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.96 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.97 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.98 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.99 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.100 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.101 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.102 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.103 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.104 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.105 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.106 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.107 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.108 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.109 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.110 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.111 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.112 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ofbkeyandinitializationvectorlength) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +Users need an ability to encrypt and decrypt column data with tenant specific keys. +Use cases include protection of sensitive column values and [GDPR] right to forget policies. +The implementation will support capabilities of the [MySQL aes_encrypt] and [MySQL aes_decrypt] +functions which encrypt and decrypt values using the [AES] (Advanced Encryption Standard) +algorithm. This functionality will enable encryption and decryption of data +accessed on remote [MySQL] servers via [MySQL Dictionary] or [MySQL Database Engine], +[MySQL Table Engine], or [MySQL Table Function]. + +## Terminology + +* **AES** - + Advanced Encryption Standard ([AES]) + +## Requirements + +### Generic + +#### RQ.SRS008.AES.Functions +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data. + +#### RQ.SRS008.AES.Functions.Compatability.MySQL +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7]. + +#### RQ.SRS008.AES.Functions.Compatability.Dictionaries +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed on remote +[MySQL] servers using [MySQL Dictionary]. + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine], + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine]. + +#### RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function]. + +#### RQ.SRS008.AES.Functions.DifferentModes +version: 1.0 + +[ClickHouse] SHALL allow different modes to be supported in a single SQL statement +using explicit function parameters. + +#### RQ.SRS008.AES.Functions.DataFromMultipleSources +version: 1.0 + +[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources +in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary], +[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function] +with possibly different encryption schemes. + +#### RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues +version: 1.0 + +[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log, +error log, and `query_log` table to prevent leakage of sensitive values. + +#### RQ.SRS008.AES.Functions.InvalidParameters +version: 1.0 + +[ClickHouse] SHALL return an error when parameters are invalid. + +#### RQ.SRS008.AES.Functions.Mismatched.Key +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched keys. + +#### RQ.SRS008.AES.Functions.Mismatched.IV +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.AAD +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.Mode +version: 1.0 + +[ClickHouse] SHALL return an error or garbage for mismatched mode. + +#### RQ.SRS008.AES.Functions.Check.Performance +version: 1.0 + +Performance of [AES] encryption functions SHALL be measured. + +#### RQ.SRS008.AES.Function.Check.Performance.BestCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the best case +scenario where there is one key, one initialization vector, and one large stream of data. + +#### RQ.SRS008.AES.Function.Check.Performance.WorstCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the worst case +where there are `N` keys, `N` initialization vectors and `N` very small streams of data. + +#### RQ.SRS008.AES.Functions.Check.Compression +version: 1.0 + +Effect of [AES] encryption on column compression SHALL be measured. + +#### RQ.SRS008.AES.Functions.Check.Compression.LowCardinality +version: 1.0 + +Effect of [AES] encryption on the compression of a column with [LowCardinality] data type +SHALL be measured. + +### Specific + +#### RQ.SRS008.AES.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt` function + +```sql +aes_encrypt(plaintext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors +version: 1.0 + +[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt` function + +```sql +aes_decrypt(ciphertext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as +the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +### MySQL Specific Functions + +#### RQ.SRS008.AES.MySQL.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function + +```sql +aes_encrypt_mysql(plaintext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_encrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function + +```sql +aes_decrypt_mysql(ciphertext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting any data type as +the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_decrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +## References + +* **GDPR:** https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +* **MySQL:** https://www.mysql.com/ +* **AES:** https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +* **ClickHouse:** https://clickhouse.tech +* **Git:** https://git-scm.com/ + +[OpenSSL]: https://www.openssl.org/ +[LowCardinality]: https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ +[MergeTree]: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/ +[MySQL Database Engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/ +[MySQL Table Engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/ +[MySQL Table Function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/ +[MySQL Dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql +[GCM]: https://en.wikipedia.org/wiki/Galois/Counter_Mode +[CTR]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR) +[CBC]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_block_chaining_(CBC) +[ECB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_(ECB) +[CFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB1]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB8]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB128]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[OFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Output_feedback_(OFB) +[GDPR]: https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +[RFC5116]: https://tools.ietf.org/html/rfc5116#section-5.1 +[MySQL]: https://www.mysql.com/ +[MySQL 5.7]: https://dev.mysql.com/doc/refman/5.7/en/ +[MySQL aes_encrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-encrypt +[MySQL aes_decrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-decrypt +[AES]: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +[ClickHouse]: https://clickhouse.tech +[GitHub repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/aes_encryption/requirements/requirements.md +[Revision history]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/aes_encryption/requirements/requirements.md +[Git]: https://git-scm.com/ +[NIST test vectors]: https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program From 88ac9bef1ba90a8570c4af1d3ee42c211b1f85c3 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 15 Sep 2020 15:45:52 +0300 Subject: [PATCH 060/441] Delete gtest_openssl.cpp --- src/Functions/tests/gtest_openssl.cpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/Functions/tests/gtest_openssl.cpp diff --git a/src/Functions/tests/gtest_openssl.cpp b/src/Functions/tests/gtest_openssl.cpp deleted file mode 100644 index e69de29bb2d..00000000000 From 951154ad8f6cea111cc64c507f11657aad87d965 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 15 Sep 2020 15:47:57 +0300 Subject: [PATCH 061/441] Update defines.h --- base/common/defines.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/common/defines.h b/base/common/defines.h index 722a1f206bf..af5981023ff 100644 --- a/base/common/defines.h +++ b/base/common/defines.h @@ -70,12 +70,10 @@ # define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) # define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address"))) # define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread"))) -# define NO_SANITIZE_MEMORY __attribute__((__no_sanitize__("memory"))) #else /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it. # define NO_SANITIZE_UNDEFINED # define NO_SANITIZE_ADDRESS # define NO_SANITIZE_THREAD -# define NO_SANITIZE_MEMORY #endif #if defined __GNUC__ && !defined __clang__ From 3d9ec963e9b2390ae3757e819f60a00fcc7f723e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 15 Sep 2020 15:49:16 +0300 Subject: [PATCH 062/441] Update query_masking_rules.xml --- programs/server/config.d/query_masking_rules.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/programs/server/config.d/query_masking_rules.xml b/programs/server/config.d/query_masking_rules.xml index 9ba77198d88..f919523472c 100644 --- a/programs/server/config.d/query_masking_rules.xml +++ b/programs/server/config.d/query_masking_rules.xml @@ -15,14 +15,5 @@ TOPSECRET.TOPSECRET [hidden] - - - hide encrypt/decrypt arguments - ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) - - \1(???) - From a93c579798738d0375bb2c0bb9e393288722c5fe Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:36:57 +0300 Subject: [PATCH 063/441] Update ColumnString.cpp --- src/Columns/ColumnString.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index d38008c44c7..16f65c2eefa 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -615,7 +615,7 @@ void ColumnString::protect() void ColumnString::validate() const { if (!offsets.empty() && offsets.back() != chars.size()) - throw Exception(fmt::format("ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size()); } } From d97426fe9ad784a8fabbe83651bd5312050ac242 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:47:43 +0300 Subject: [PATCH 064/441] Update FunctionsAES.cpp --- src/Functions/FunctionsAES.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 0d459b6095f..120be3749f1 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -25,6 +27,7 @@ void onError(std::string error_message) StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key) { + assert(cipher_key_size <= EVP_MAX_KEY_LENGTH); memcpy(folded_key.data(), key.data, cipher_key_size); for (size_t i = cipher_key_size; i < key.size; ++i) From 60f91332b3732f46d6fbcc48f050e2baa2bd8901 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:52:01 +0300 Subject: [PATCH 065/441] Update FunctionsAES.h --- src/Functions/FunctionsAES.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index fc9b9ef6f40..24e797f54cf 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -19,12 +19,12 @@ #include #include -#include #include #include #include + namespace DB { namespace ErrorCodes From 200bc9b9b2a390db7ecf4cc363cd047dde7d73df Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:57:26 +0300 Subject: [PATCH 066/441] Update FunctionsAES.h --- src/Functions/FunctionsAES.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 24e797f54cf..2bdcb23611c 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -7,10 +7,8 @@ #if USE_SSL #include #include -#include #include #include - #include #include From 88738cd6100561aec41f3faa7cb8d2b4050ec20d Mon Sep 17 00:00:00 2001 From: 20018712 <20018712@sn.suning.ad> Date: Thu, 17 Sep 2020 07:27:55 +0800 Subject: [PATCH 067/441] =?UTF-8?q?when=20using=20copier=EF=BC=8Ccolumn=20?= =?UTF-8?q?name=20,as=20sort=20key,with=20digital=20number=20as=20beginnin?= =?UTF-8?q?g,will=20cause=20a=20syntax=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- programs/copier/Internals.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 12da07a772a..4c767f17d5f 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -208,7 +208,7 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) if (!primary_key_columns_set.emplace(pk_column).second) throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); - primary_key_columns.push_back(pk_column); + primary_key_columns.push_back("`" + pk_column + "`"); } } From e101c912aac97f95ab112956f51e1d85e866d2b2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 17:19:10 +0300 Subject: [PATCH 068/441] Update Internals.cpp --- programs/copier/Internals.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 4c767f17d5f..9008767775b 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -208,7 +208,7 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) if (!primary_key_columns_set.emplace(pk_column).second) throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); - primary_key_columns.push_back("`" + pk_column + "`"); + primary_key_columns.push_back(backQuote(pk_column)); } } From c1e9ce2cf2fb0b45f1644cf8bd79c819ce4ae298 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 17:38:06 +0300 Subject: [PATCH 069/441] Remove questionable code --- programs/copier/Internals.cpp | 2 -- programs/copier/TaskTableAndShard.h | 5 ++++- src/DataTypes/NestedUtils.cpp | 11 ----------- src/DataTypes/NestedUtils.h | 2 -- src/Storages/MergeTree/MergeTreeData.cpp | 3 ++- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 12da07a772a..dbd8667eba9 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -189,13 +189,11 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) ErrorCodes::BAD_ARGUMENTS); Names primary_key_columns; - Names sorting_key_columns; NameSet primary_key_columns_set; for (size_t i = 0; i < sorting_key_size; ++i) { String sorting_key_column = sorting_key_expr_list->children[i]->getColumnName(); - sorting_key_columns.push_back(sorting_key_column); if (i < primary_key_size) { diff --git a/programs/copier/TaskTableAndShard.h b/programs/copier/TaskTableAndShard.h index 11ceffd12cd..37c117139c7 100644 --- a/programs/copier/TaskTableAndShard.h +++ b/programs/copier/TaskTableAndShard.h @@ -6,6 +6,9 @@ #include +#include +#include + namespace DB { @@ -268,7 +271,7 @@ inline TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConf ParserStorage parser_storage; engine_push_ast = parseQuery(parser_storage, engine_push_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); - primary_key_comma_separated = Nested::createCommaSeparatedStringFrom(extractPrimaryKeyColumnNames(engine_push_ast)); + primary_key_comma_separated = boost::algorithm::join(ext::map(extractPrimaryKeyColumnNames(engine_push_ast), backQuote), ", "); engine_push_zk_path = extractReplicatedTableZookeeperPath(engine_push_ast); } diff --git a/src/DataTypes/NestedUtils.cpp b/src/DataTypes/NestedUtils.cpp index 7dd7bb03c20..0537fa5cdc1 100644 --- a/src/DataTypes/NestedUtils.cpp +++ b/src/DataTypes/NestedUtils.cpp @@ -70,17 +70,6 @@ std::pair splitName(const std::string & name) return {{ begin, first_end }, { second_begin, end }}; } -std::string createCommaSeparatedStringFrom(const Names & names) -{ - std::ostringstream ss; - if (!names.empty()) - { - std::copy(names.begin(), std::prev(names.end()), std::ostream_iterator(ss, ", ")); - ss << names.back(); - } - return ss.str(); -} - std::string extractTableName(const std::string & nested_name) { diff --git a/src/DataTypes/NestedUtils.h b/src/DataTypes/NestedUtils.h index 4be3e69edfd..3039fd7f118 100644 --- a/src/DataTypes/NestedUtils.h +++ b/src/DataTypes/NestedUtils.h @@ -13,8 +13,6 @@ namespace Nested std::pair splitName(const std::string & name); - std::string createCommaSeparatedStringFrom(const Names & names); - /// Returns the prefix of the name to the first '.'. Or the name is unchanged if there is no dot. std::string extractTableName(const std::string & nested_name); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 5969ecc5baf..64a5004c954 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -49,6 +49,7 @@ #include #include +#include #include #include @@ -632,7 +633,7 @@ void MergeTreeData::MergingParams::check(const StorageInMemoryMetadata & metadat std::back_inserter(names_intersection)); if (!names_intersection.empty()) - throw Exception("Columns: " + Nested::createCommaSeparatedStringFrom(names_intersection) + + throw Exception("Columns: " + boost::algorithm::join(names_intersection, ", ") + " listed both in columns to sum and in partition key. That is not allowed.", ErrorCodes::BAD_ARGUMENTS); } } From 08668de2c16c98dd0de6c46975dc32530886ecbd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 17 Sep 2020 17:39:22 +0300 Subject: [PATCH 070/441] Remove previous modification --- programs/copier/Internals.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 902a8cd8cf1..dbd8667eba9 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -206,7 +206,7 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) if (!primary_key_columns_set.emplace(pk_column).second) throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); - primary_key_columns.push_back(backQuote(pk_column)); + primary_key_columns.push_back(pk_column); } } From 03481f7a3a190a8a10d6d6d1fb412da6b20b651a Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 21 Sep 2020 02:51:38 +0400 Subject: [PATCH 071/441] Synchronize the code with respect to IAccessStorage::login() functionality --- src/Access/AccessControlManager.cpp | 9 +- src/Access/IAccessStorage.cpp | 12 --- src/Access/IAccessStorage.h | 7 -- src/Access/LDAPAccessStorage.cpp | 134 ++++++++++++++------------- src/Access/LDAPAccessStorage.h | 10 +- src/Access/MultipleAccessStorage.cpp | 17 ---- src/Access/MultipleAccessStorage.h | 1 - 7 files changed, 75 insertions(+), 115 deletions(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 9a9bca7734a..f5f1e4b3de8 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -257,9 +257,7 @@ void AccessControlManager::addMemoryStorage(const String & storage_name_) void AccessControlManager::addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) { - auto storage = std::make_shared(storage_name_); - storage->setConfiguration(this, config_, prefix_); - addStorage(storage); + addStorage(std::make_shared(storage_name_, this, config_, prefix_)); } @@ -279,8 +277,7 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( String prefix = key + "." + key_in_user_directories; String type = key_in_user_directories; - const size_t bracket_pos = type.find('['); - if (bracket_pos != String::npos) + if (size_t bracket_pos = type.find('['); bracket_pos != String::npos) type.resize(bracket_pos); if ((type == "users_xml") || (type == "users_config")) type = UsersConfigAccessStorage::STORAGE_TYPE; @@ -310,8 +307,6 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( } else if (type == LDAPAccessStorage::STORAGE_TYPE) { - if (bracket_pos != String::npos) - throw Exception("Duplicate storage type '" + type + "' at " + prefix + " in config", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); addLDAPStorage(name, config, prefix); } else diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index fbc95342972..e5170221e18 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -153,12 +153,6 @@ std::vector IAccessStorage::find(EntityType type, const Strings & names) c } -std::optional IAccessStorage::findOrGenerate(EntityType type, const String & name) const -{ - return findOrGenerateImpl(type, name); -} - - UUID IAccessStorage::getID(EntityType type, const String & name) const { auto id = findImpl(type, name); @@ -489,12 +483,6 @@ Poco::Logger * IAccessStorage::getLogger() const } -std::optional IAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const -{ - return findImpl(type, name); -} - - void IAccessStorage::throwNotFound(const UUID & id) const { throw Exception(outputID(id) + " not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index c47ab608c20..5a86e817fb2 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -53,12 +53,6 @@ public: template std::vector find(const Strings & names) const { return find(EntityClassT::TYPE, names); } - /// Searches for an entity with specified type and name. Returns std::nullopt if not found and cannot be generated. - std::optional findOrGenerate(EntityType type, const String & name) const; - - template - std::optional findOrGenerate(const String & name) const { return findOrGenerate(EntityClassT::TYPE, name); } - /// Searches for an entity with specified name and type. Throws an exception if not found. UUID getID(EntityType type, const String & name) const; @@ -158,7 +152,6 @@ public: protected: virtual std::optional findImpl(EntityType type, const String & name) const = 0; - virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const; virtual std::vector findAllImpl(EntityType type) const = 0; virtual bool existsImpl(const UUID & id) const = 0; virtual AccessEntityPtr readImpl(const UUID & id) const = 0; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 59f061c51a5..ae61d912ebe 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -6,6 +6,10 @@ #include #include #include +#include +#include +#include +#include namespace DB @@ -16,9 +20,10 @@ namespace ErrorCodes } -LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_) +LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix) : IAccessStorage(storage_name_) { + setConfiguration(access_control_manager_, config, prefix); } @@ -62,12 +67,6 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m } -bool LDAPAccessStorage::isConfiguredNoLock() const -{ - return !ldap_server.empty() &&/* !roles.empty() &&*/ access_control_manager; -} - - void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) { auto role_ptr = typeid_cast>(entity); @@ -122,9 +121,17 @@ const char * LDAPAccessStorage::getStorageType() const } -bool LDAPAccessStorage::isStorageReadOnly() const +String LDAPAccessStorage::getStorageParamsJSON() const { - return true; + Poco::JSON::Object params_json; + + params_json.set("server", ldap_server); + params_json.set("roles", default_role_names); + + std::ostringstream oss; + Poco::JSON::Stringifier::stringify(params_json, oss); + + return oss.str(); } @@ -134,62 +141,6 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & } -std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const -{ - if (type == EntityType::USER) - { - std::scoped_lock lock(mutex); - - // Return the id immediately if we already have it. - const auto id = memory_storage.find(type, name); - if (id.has_value()) - return id; - - if (!isConfiguredNoLock()) - return {}; - - // Stop if entity exists anywhere else, to avoid generating duplicates. - const auto * this_base = dynamic_cast(this); - const auto storages = access_control_manager->getStoragesPtr(); - for (const auto & storage : *storages) - { - if (storage.get() != this_base && storage->find(type, name)) - return {}; - } - - // Entity doesn't exist. We are going to create one. - const auto user = std::make_shared(); - user->setName(name); - user->authentication = Authentication(Authentication::Type::LDAP_SERVER); - user->authentication.setServerName(ldap_server); - - for (const auto& role_name : default_role_names) - { - std::optional role_id; - - try - { - role_id = access_control_manager->find(role_name); - if (!role_id) - throw Exception("Retrieved role info is empty", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); - } - catch (...) - { - tryLogCurrentException(getLogger(), "Unable to retrieve role '" + role_name + "' info from access storage '" + access_control_manager->getStorageName() + "'"); - return {}; - } - - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } - - return memory_storage.insert(user); - } - - return memory_storage.find(type, name); -} - - std::vector LDAPAccessStorage::findAllImpl(EntityType type) const { return memory_storage.findAll(type); @@ -262,4 +213,57 @@ bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const { return memory_storage.hasSubscription(type); } + +UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const +{ + std::scoped_lock lock(mutex); + try + { + auto id = memory_storage.find(user_name); + if (id) + { + // We try to re-authenticate the existing user, and if not successful, we will remove it, since that would mean + // something changed and the user we authenticated previously cannot be authenticated anymore. + auto user = memory_storage.tryRead(*id); + try + { + if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) + return *id; + } + catch (...) + { + memory_storage.remove(*id); + throw; + } + memory_storage.remove(*id); + } + else + { + // User does not exist, so we create one, and will add it if authentication is successful. + auto user = std::make_shared(); + user->setName(user_name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + if (isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) + { + for (const auto& role_name : default_role_names) + { + std::optional role_id = access_control_manager->find(role_name); + if (!role_id) + throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); + roles_of_interest.insert(role_id.value()); + user->granted_roles.grant(role_id.value()); + } + return memory_storage.insert(user); + } + } + } + catch (...) + { + tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + access_control_manager->getStorageName() + "'"); + } + throwCannotAuthenticate(user_name); +} + } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index d52056fe947..35444944ec6 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -28,18 +28,15 @@ class LDAPAccessStorage : public IAccessStorage public: static constexpr char STORAGE_TYPE[] = "ldap"; - explicit LDAPAccessStorage(const String & storage_name_ = STORAGE_TYPE); + explicit LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); virtual ~LDAPAccessStorage() override = default; - void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix = ""); - public: // IAccessStorage implementations. virtual const char * getStorageType() const override; - virtual bool isStorageReadOnly() const override; + virtual String getStorageParamsJSON() const override; private: // IAccessStorage implementations. virtual std::optional findImpl(EntityType type, const String & name) const override; - virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const override; virtual std::vector findAllImpl(EntityType type) const override; virtual bool existsImpl(const UUID & id) const override; virtual AccessEntityPtr readImpl(const UUID & id) const override; @@ -52,9 +49,10 @@ private: // IAccessStorage implementations. virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; virtual bool hasSubscriptionImpl(const UUID & id) const override; virtual bool hasSubscriptionImpl(EntityType type) const override; + virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; private: - bool isConfiguredNoLock() const; + void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); mutable std::recursive_mutex mutex; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index e613279a466..8ddc7410d8d 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -104,23 +104,6 @@ std::optional MultipleAccessStorage::findImpl(EntityType type, const Strin } -std::optional MultipleAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const -{ - auto storages = getStoragesInternal(); - for (const auto & storage : *storages) - { - auto id = storage->findOrGenerate(type, name); - if (id) - { - std::lock_guard lock{mutex}; - ids_cache.set(*id, storage); - return id; - } - } - return {}; -} - - std::vector MultipleAccessStorage::findAllImpl(EntityType type) const { std::vector all_ids; diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 088e1e82e05..36551f1cbc8 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -35,7 +35,6 @@ public: protected: std::optional findImpl(EntityType type, const String & name) const override; - std::optional findOrGenerateImpl(EntityType type, const String & name) const override; std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; From 2c6b6673f2d7350b59a55838a45d429e20af7090 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 21 Sep 2020 02:57:34 +0400 Subject: [PATCH 072/441] Remove extra declaration --- src/Access/AccessControlManager.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index f5f1e4b3de8..56d225f64f4 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -24,7 +24,6 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_ELEMENT_IN_CONFIG; - extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int UNKNOWN_SETTING; } From 536c8c2416464bbcfe09227fb30538fc79dcfbbd Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Fri, 25 Sep 2020 11:39:29 +0300 Subject: [PATCH 073/441] Russian text updated. --- docs/ru/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 1396f43a01c..878616de8ca 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1882,7 +1882,7 @@ SELECT range(number) FROM system.numbers LIMIT 5 FORMAT PrettyCompactNoEscapes; ## cast_keep_nullable {#cast_keep_nullable} -Включает или отключает финальное преобразование аргумента функции [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) к типу `Nullable`. +Включает или отключает сохранение типа `Nullable` для аргумента функции [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast). Если настройка включена, то функция `CAST(something_nullable AS Type)` возвращает `Nullable(Type)`. From a1f4d38019a408c2a840df1dc877ea04983cb089 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 29 Sep 2020 12:44:28 +0300 Subject: [PATCH 074/441] Add ability to run tests multiple time in stateless image --- docker/test/stateless/Dockerfile | 3 +++ docker/test/stateless/run.sh | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 516d8d5842b..c7529686cbe 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -33,5 +33,8 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ENV NUM_TRIES=1 +ENV MAX_RUN_TIME=0 #unlimited + COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 9f2bb9bf62d..e736ef835c7 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -17,4 +17,13 @@ if cat /usr/bin/clickhouse-test | grep -q -- "--use-skip-list"; then SKIP_LIST_OPT="--use-skip-list" fi -clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt +function run_tests() +{ + for i in $(seq 1 $NUM_TRIES); do + clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt + done +} + +export -f run_tests + +timeout $MAX_RUN_TIME bash -c run_tests ||: From 162541217fb165ca0f273f6019e91e417b2755b6 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 29 Sep 2020 22:58:09 +0200 Subject: [PATCH 075/441] Updating tests to match changes in the behavior. --- .../tests/authentications.py | 2 +- .../tests/external_user_directory_config.py | 14 ++++- .../tests/server_config.py | 53 ++++++------------- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 76e55996f21..e4df466b1ea 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -506,4 +506,4 @@ def feature(self, servers=None, server=None, node="clickhouse1"): with rbac_roles("ldap_role") as roles: with ldap_external_user_directory(server=server, roles=roles, restart=True): for scenario in loads(current_module(), Scenario): - scenario(server=server) + Scenario(test=scenario, flags=TE)(server=server) diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py index d29d0124dc2..d95a4a674a1 100644 --- a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -36,7 +36,17 @@ def more_than_one_user_directory(self, timeout=20): with ldap_servers(servers): with rbac_roles(role) as roles: config = create_entries_ldap_external_user_directory_config_content(entries) - invalid_ldap_external_user_directory_config(server=None, roles=None, message=message, timeout=timeout, config=config) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {users[0]['username']} authenticated using openldap1"): + current().context.node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I login as {users[1]['username']} authenticated using openldap2"): + current().context.node.query(f"SELECT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + + @TestScenario @Requirements( @@ -132,7 +142,7 @@ def invalid_server(self, timeout=20): with When(f"I login as {user['username']} and execute query"): current().context.node.query("SELECT 1", settings=[("user", user["username"]), ("password", user["password"])], - exitcode=36, message="DB::Exception: LDAP server 'openldap2' is not configured") + exitcode=4, message="DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.") @TestScenario @Requirements( diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py index b02b70eb6a7..5df343b53df 100644 --- a/tests/testflows/ldap/external_user_directory/tests/server_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -12,7 +12,7 @@ from ldap.authentication.tests.common import invalid_server_config RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name("1.0") ) -def empty_server_name(self, timeout=20): +def empty_server_name(self, timeout=60): """Check that empty string as a server name is not allowed. """ servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", @@ -32,7 +32,7 @@ def invalid_host(self): servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server" + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "foo", *users) @@ -41,7 +41,7 @@ def invalid_host(self): RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") ) -def empty_host(self, tail=20, timeout=20): +def empty_host(self, tail=20, timeout=60): """Check that server returns an error when LDAP server host value is empty. """ @@ -49,31 +49,15 @@ def empty_host(self, tail=20, timeout=20): message = "DB::Exception: Empty 'host' entry" servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." - }] - with Given("I prepare the error log by writting empty lines into it"): - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) - - with ldap_servers(servers): - with Then("server shall fail to merge the new config"): - started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break - time.sleep(1) - assert exitcode == 0, error() + invalid_server_config(servers, message=message, tail=16, timeout=timeout) @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") ) -def missing_host(self, tail=20, timeout=20): +def missing_host(self, tail=20, timeout=60): """Check that server returns an error when LDAP server host is missing. """ @@ -111,7 +95,7 @@ def invalid_port(self): servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server." + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap1", *users) @@ -130,7 +114,7 @@ def invalid_auth_dn_prefix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap1", *users) @@ -148,7 +132,7 @@ def invalid_auth_dn_suffix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap1", *users) @@ -157,18 +141,15 @@ def invalid_auth_dn_suffix(self): RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0") ) -def invalid_enable_tls_value(self): +def invalid_enable_tls_value(self, timeout=60): """Check that server returns an error when enable_tls option has invalid value. """ + message = "Syntax error: Cannot convert to boolean: foo" servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap1' is not configured" - }] - login(servers, "openldap1", *users) + invalid_server_config(servers, message=message, tail=17, timeout=timeout) @TestScenario @Requirements( @@ -188,7 +169,7 @@ def invalid_tls_require_cert_value(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap2' is not configured" + "exitcode": 4, "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap2", *users) @@ -208,8 +189,8 @@ def empty_ca_cert_dir(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, "openldap2", *users) @@ -229,8 +210,8 @@ def empty_ca_cert_file(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "Received from localhost:9000. DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain)" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap2", *users) @@ -302,4 +283,4 @@ def feature(self, node="clickhouse1"): """ self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): - scenario() \ No newline at end of file + scenario() From 581a14be50eba7c57bdf724e75ad366ad4498c5e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 30 Sep 2020 22:48:32 +0200 Subject: [PATCH 076/441] Adding SRS source for LDAP external user directory. Small updates to helpers/cluster.py. Updating link in the ldap/authentication/requirements/requirements.md. --- tests/testflows/helpers/cluster.py | 6 +- .../requirements/requirements.md | 7 +- .../requirements/requirements.md | 712 ++++++++++++++++++ .../requirements/requirements.py | 104 ++- .../tests/authentications.py | 2 +- .../tests/external_user_directory_config.py | 2 +- 6 files changed, 813 insertions(+), 20 deletions(-) create mode 100644 tests/testflows/ldap/external_user_directory/requirements/requirements.md diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 761acce40d3..8f386691638 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -296,12 +296,12 @@ class Cluster(object): :param steps: don't break command into steps, default: True """ debug(f"command() {node}, {command}") - with By("executing command", description=command) if steps else NullStep(): + with By("executing command", description=command, format_description=False) if steps else NullStep(): r = self.bash(node)(command, *args, **kwargs) if exitcode is not None: - with Then(f"exitcode should be {exitcode}") if steps else NullStep(): + with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep(): assert r.exitcode == exitcode, error(r.output) if message is not None: - with Then(f"output should contain message", description=message) if steps else NullStep(): + with Then(f"output should contain message", description=message, format_description=False) if steps else NullStep(): assert message in r.output, error(r.output) return r diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index 6d787670138..d322db70330 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -524,9 +524,6 @@ used to authenticate users using an [LDAP] server. ## References * **ClickHouse:** https://clickhouse.tech -* **GitHub repository:** https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/requirements/requirements.md -* **Revision history:** https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/requirements/requirements.md -* **Git:** https://git-scm.com/ [Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind [Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind @@ -539,6 +536,6 @@ used to authenticate users using an [LDAP] server. [LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol [ClickHouse]: https://clickhouse.tech [GitHub]: https://github.com -[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/requirements/requirements.md -[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/requirements/requirements.md +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md [Git]: https://git-scm.com/ diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.md b/tests/testflows/ldap/external_user_directory/requirements/requirements.md new file mode 100644 index 00000000000..37a8be04b66 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.md @@ -0,0 +1,712 @@ +# SRS-009 ClickHouse LDAP External User Directory +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) + * 3.1 [LDAP](#ldap) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [User Authentication](#user-authentication) + * 4.1.1.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication](#rqsrs-009ldapexternaluserdirectoryauthentication) + * 4.1.1.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories](#rqsrs-009ldapexternaluserdirectorymultipleuserdirectories) + * 4.1.1.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup](#rqsrs-009ldapexternaluserdirectorymultipleuserdirectorieslookup) + * 4.1.1.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers](#rqsrs-009ldapexternaluserdirectoryusersauthenticationnewusers) + * 4.1.1.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers](#rqsrs-009ldapexternaluserdirectoryauthenticationdeletedusers) + * 4.1.1.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid](#rqsrs-009ldapexternaluserdirectoryauthenticationvalid) + * 4.1.1.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid](#rqsrs-009ldapexternaluserdirectoryauthenticationinvalid) + * 4.1.1.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamechanged) + * 4.1.1.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordchanged) + * 4.1.1.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart](#rqsrs-009ldapexternaluserdirectoryauthenticationldapserverrestart) + * 4.1.1.11 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart](#rqsrs-009ldapexternaluserdirectoryauthenticationclickhouseserverrestart) + * 4.1.1.12 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel](#rqsrs-009ldapexternaluserdirectoryauthenticationparallel) + * 4.1.1.13 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelvalidandinvalid) + * 4.1.1.14 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelmultipleservers) + * 4.1.1.15 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly](#rqsrs-009ldapexternaluserdirectoryauthenticationparallellocalonly) + * 4.1.1.16 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP](#rqsrs-009ldapexternaluserdirectoryauthenticationparallellocalandmultipleldap) + * 4.1.1.17 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelsameuser) + * 4.1.2 [Connection](#connection) + * 4.1.2.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText](#rqsrs-009ldapexternaluserdirectoryconnectionprotocolplaintext) + * 4.1.2.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltls) + * 4.1.2.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS](#rqsrs-009ldapexternaluserdirectoryconnectionprotocolstarttls) + * 4.1.2.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificatevalidation) + * 4.1.2.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificateselfsigned) + * 4.1.2.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificatespecificcertificationauthority) + * 4.1.2.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismanonymous) + * 4.1.2.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismunauthenticated) + * 4.1.2.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismnamepassword) + * 4.1.2.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationunreachableserver) + * 4.2 [Specific](#specific) + * 4.2.1 [User Discovery](#user-discovery) + * 4.2.1.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority](#rqsrs-009ldapexternaluserdirectoryuserslookuppriority) + * 4.2.2 [Roles](#roles) + * 4.2.2.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed](#rqsrs-009ldapexternaluserdirectoryroleremoved) + * 4.2.2.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges](#rqsrs-009ldapexternaluserdirectoryroleremovedprivileges) + * 4.2.2.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges](#rqsrs-009ldapexternaluserdirectoryrolereaddedprivileges) + * 4.2.2.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New](#rqsrs-009ldapexternaluserdirectoryrolenew) + * 4.2.2.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege](#rqsrs-009ldapexternaluserdirectoryrolenewprivilege) + * 4.2.2.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege](#rqsrs-009ldapexternaluserdirectoryroleremovedprivilege) + * 4.2.3 [Configuration](#configuration) + * 4.2.3.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationserverinvalid) + * 4.2.3.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition](#rqsrs-009ldapexternaluserdirectoryconfigurationserverdefinition) + * 4.2.3.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name](#rqsrs-009ldapexternaluserdirectoryconfigurationservername) + * 4.2.3.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host](#rqsrs-009ldapexternaluserdirectoryconfigurationserverhost) + * 4.2.3.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port](#rqsrs-009ldapexternaluserdirectoryconfigurationserverport) + * 4.2.3.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserverportdefault) + * 4.2.3.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnprefix) + * 4.2.3.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnsuffix) + * 4.2.3.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnvalue) + * 4.2.3.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletls) + * 4.2.3.11 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsdefault) + * 4.2.3.12 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsno) + * 4.2.3.13 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsyes) + * 4.2.3.14 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsstarttls) + * 4.2.3.15 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversion) + * 4.2.3.16 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversionvalues) + * 4.2.3.17 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversiondefault) + * 4.2.3.18 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecert) + * 4.2.3.19 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsdefault) + * 4.2.3.20 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsdemand) + * 4.2.3.21 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsallow) + * 4.2.3.22 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionstry) + * 4.2.3.23 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsnever) + * 4.2.3.24 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscertfile) + * 4.2.3.25 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlskeyfile) + * 4.2.3.26 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertdir) + * 4.2.3.27 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertfile) + * 4.2.3.28 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsciphersuite) + * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) + * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) + * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) + * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) + * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) + * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) + * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) + * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) + * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) + * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) + * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) + * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) + * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) + * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) + * 4.2.4 [Authentication](#authentication) + * 4.2.4.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameempty) + * 4.2.4.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamelong) + * 4.2.4.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameutf8) + * 4.2.4.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordempty) + * 4.2.4.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordlong) + * 4.2.4.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordutf8) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +The [QA-SRS007 ClickHouse Authentication of Users via LDAP] enables support for authenticating +users using an [LDAP] server. This requirements specifications add addition functionality +for integrating [LDAP] with [ClickHouse]. + +This document will cover requirements to allow authenticatoin of users stored in the +external user discovery using an [LDAP] server without having to explicitly define users in [ClickHouse]'s +`users.xml` configuration file. + +## Terminology + +### LDAP + +* Lightweight Directory Access Protocol + +## Requirements + +### Generic + +#### User Authentication + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication +version: 1.0 + +[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories +version: 1.0 + +[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup +version: 1.0 + +[ClickHouse] SHALL attempt to authenticate external [LDAP] user +using [LDAP] external user directory in the same order +in which user directories are specified in the `config.xml` file. +If a user cannot be authenticated using the first [LDAP] external user directory +then the next user directory in the list SHALL be used. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers +version: 1.0 + +[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server +as soon as they are added to the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers +version: 1.0 + +[ClickHouse] SHALL not allow authentication of users that +were previously defined only on the [LDAP] server but were removed +from the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of external [LDAP] users +authenticated using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users defined only locally +when one or more [LDAP] external user directories are specified in the configuration file. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users +authenticated using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user +authenticated using the same [LDAP] external user directory. + +#### Connection + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol +while connecting to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol +while connecting to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable +when using [LDAP] external user directory. + +### Specific + +#### User Discovery + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority +version: 2.0 + +[ClickHouse] SHALL lookup user presence in the same order +as user directories are defined in the `config.xml`. + +#### Roles + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed +version: 1.0 + +[ClickHouse] SHALL reject authentication attempt if any of the roles that are specified in the configuration +of the external user directory are not defined at the time of the authentication attempt +with an exception that if a user was able to authenticate in past and its internal user object was created and cached +then the user SHALL be able to authenticate again, even if one of the roles is missing. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges +version: 1.0 + +[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP +users authenticated using external user directory if it is removed +including currently cached users that are still able to authenticated where the removed +role is specified in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges +version: 1.0 + +[ClickHouse] SHALL reassign the role and add the privileges provided by the role +when it is re-added after removal for all LDAP users authenticated using external user directory +including any cached users where the re-added role was specified in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New +version: 1.0 + +[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP +users authenticated using external user directory unless the role is specified +in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege +version: 1.0 + +[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory +including cached users when new privilege is added to one of the roles specified +in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege +version: 1.0 + +[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory +including cached users when privilege is removed from all the roles specified +in the configuration of the external user directory. + +#### Configuration + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition +version: 1.0 + +[ClickHouse] SHALL support using the [LDAP] servers defined in the +`ldap_servers` section of the `config.xml` as the server to be used +for a external user directory that uses an [LDAP] server as a source of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory +version: 1.0 + +[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml` +that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne +version: 2.0 + +[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml` +that SHALL allow to define more than one external user directory that use an [LDAP] server as a source +of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax +version: 1.0 + +[ClickHouse] SHALL support `` section with the following syntax + +```xml + + + + my_ldap_server + + + + + + + +``` + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server +version: 1.0 + +[ClickHouse] SHALL support `server` parameter in the `` sub-section in the `` +section of the `config.xml` that SHALL specify one of LDAP server names +defined in `` section. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty +version: 1.0 + +[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the `` +is empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing +version: 1.0 + +[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the `` +is missing. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne +version: 1.0 + +[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the `` +if more than one `server` parameter is defined in the configuration. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the server specified as the value of the `` +parameter is not defined. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles +version: 1.0 + +[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the `` +section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL +be assigned to all users retrieved from the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne +version: 1.0 + +[ClickHouse] SHALL only use the first definitition of the `roles` parameter +in the `` sub-section in the `` +if more than one `roles` parameter is defined in the configuration. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the role specified in the `` +parameter does not exist locally. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty +version: 1.0 + +[ClickHouse] SHALL not allow users authenticated using LDAP external user directory +to perform any action if the `roles` parameter in the `` sub-section in the `` +section is empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing +version: 1.0 + +[ClickHouse] SHALL not allow users authenticated using LDAP external user directory +to perform any action if the `roles` parameter in the `` sub-section in the `` +section is missing. + +#### Authentication + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users when using [LDAP] external user directory. + +## References + +* **Access Control and Account Management**: https://clickhouse.tech/docs/en/operations/access-rights/ +* **LDAP**: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +* **ClickHouse:** https://clickhouse.tech + +[SRS]: #srs +[Access Control and Account Management]: https://clickhouse.tech/docs/en/operations/access-rights/ +[SRS-007 ClickHouse Authentication of Users via LDAP]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py index 7e3ced037fa..bb21f7d3eb9 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.py +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200827.1211600. +# document by TestFlows v1.6.200929.1033606. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Requirement @@ -18,6 +18,36 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication = Requirement( link=None ) +RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories_Lookup = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL attempt to authenticate external [LDAP] user\n' + 'using [LDAP] external user directory in the same order\n' + 'in which user directories are specified in the `config.xml` file.\n' + 'If a user cannot be authenticated using the first [LDAP] external user directory\n' + 'then the next user directory in the list SHALL be used.\n' + ), + link=None + ) + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers', version='1.0', @@ -163,6 +193,62 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid = link=None ) +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of external [LDAP] users\n' + 'authenticated using multiple [LDAP] external user directories.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of users defined only locally\n' + 'when one or more [LDAP] external user directories are specified in the configuration file.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users\n' + 'authenticated using multiple [LDAP] external user directories.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user\n' + 'authenticated using the same [LDAP] external user directory.\n' + ), + link=None + ) + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText', version='1.0', @@ -306,17 +392,14 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServe RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority', - version='1.0', + version='2.0', priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL lookup user presence in the following priority:\n' - '\n' - '1. access control\n' - '2. `users.xml`\n' - '3. LDAP\n' + '[ClickHouse] SHALL lookup user presence in the same order\n' + 'as user directories are defined in the `config.xml`.\n' ), link=None ) @@ -863,14 +946,15 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory = Re RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne', - version='1.0', + version='2.0', priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only use the first `` sub-section in the `` section of the `config.xml`\n' - 'if more than one `` sub-sections are present.\n' + '[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml`\n' + 'that SHALL allow to define more than one external user directory that use an [LDAP] server as a source\n' + 'of user definitions.\n' ), link=None ) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index e4df466b1ea..55290d1ada9 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -450,7 +450,7 @@ def empty_username_and_empty_password(self, server=None): @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("2.0") ) def user_lookup_priority(self, server): """Check that users are looked up in the same priority diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py index d95a4a674a1..b5677eba4b2 100644 --- a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -5,7 +5,7 @@ from ldap.external_user_directory.requirements import * @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("2.0") ) def more_than_one_user_directory(self, timeout=20): """Check when more than one LDAP user directory is From 5af69d2c17e82c8d541dc54d8f8499f7654dbe61 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 20 Sep 2020 16:28:19 +0300 Subject: [PATCH 077/441] Add a test for jemalloc percpu_arena --- .../01502_jemalloc_percpu_arena.reference | 1 + .../01502_jemalloc_percpu_arena.sh | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference create mode 100755 tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh new file mode 100755 index 00000000000..06f7d38af94 --- /dev/null +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +ncpus="$(getconf _NPROCESSORS_ONLN)" + +# to hit possible issues even in unbundled builds: +# (although likiley jemalloc will be compiled with NDEBUG there) +export MALLOC_CONF=percpu_arena:percpu + +# Regression for: +# +# $ taskset --cpu-list 8 ./clickhouse local -q 'select 1' +# : ../contrib/jemalloc/src/jemalloc.c:321: Failed assertion: "ind <= narenas_total_get()" +# Aborted (core dumped) +taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' +# just in case something more complicated +taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' From 7f776a434453672ca01bead5ca56ae6c79e497e8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 20 Sep 2020 16:35:06 +0300 Subject: [PATCH 078/441] Update jemalloc to inculde fix for percpu_arena This fix is the same as from [1], but backported to 5.2.1. [1]: https://github.com/jemalloc/jemalloc/pull/1939 --- contrib/jemalloc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/jemalloc b/contrib/jemalloc index 026764f1999..93e27e435ca 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit 026764f19995c53583ab25a3b9c06a2fd74e4689 +Subproject commit 93e27e435cac846028da20cd9b0841fbc9110bd2 From e89a56969f0719603a87f8c279272655476599d2 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Wed, 30 Sep 2020 23:16:27 +0000 Subject: [PATCH 079/441] first part --- src/Core/Settings.h | 1 + src/Storages/MergeTree/MergeTreePartInfo.h | 3 + .../MergeTree/ReplicatedMergeTreeBlockEntry.h | 66 ++++++++++++++++ .../ReplicatedMergeTreeBlockOutputStream.cpp | 62 +++++++++++---- .../ReplicatedMergeTreeBlockOutputStream.h | 2 + .../ReplicatedMergeTreeCleanupThread.cpp | 4 + .../ReplicatedMergeTreePartHeader.cpp | 15 ++++ .../MergeTree/ReplicatedMergeTreePartHeader.h | 10 ++- .../ReplicatedMergeTreeQuorumEntry.h | 28 ++----- .../ReplicatedMergeTreeQuorumStatusEntry.h | 76 +++++++++++++++++++ .../ReplicatedMergeTreeRestartingThread.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 66 ++++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 6 +- 13 files changed, 283 insertions(+), 60 deletions(-) create mode 100644 src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h create mode 100644 src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 9449cd571a1..32d6b50f647 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -158,6 +158,7 @@ class IColumn; \ M(UInt64, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled.", 0) \ M(Milliseconds, insert_quorum_timeout, 600000, "", 0) \ + M(Bool, insert_quorum_parallel, false, "For quorum INSERT queries - enable to make parallel inserts without linearizability", 0) \ M(UInt64, select_sequential_consistency, 0, "For SELECT queries from the replicated table, throw an exception if the replica does not have a chunk written with the quorum; do not read the parts that have not yet been written with the quorum.", 0) \ M(UInt64, table_function_remote_max_addresses, 1000, "The maximum number of different shards and the maximum number of replicas of one shard in the `remote` function.", 0) \ M(Milliseconds, read_backoff_min_latency_ms, 1000, "Setting to reduce the number of threads in case of slow reads. Pay attention only to reads that took at least that much time.", 0) \ diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index 416308861b7..d538418c29b 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -18,6 +19,8 @@ struct MergeTreePartInfo Int64 max_block = 0; UInt32 level = 0; Int64 mutation = 0; /// If the part has been mutated or contains mutated parts, is equal to mutation version number. + std::optional block_id; /// hex if write with quorum and min_block == max_block +/// shouldn't be here... MergeTreePartInfo() = default; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h new file mode 100644 index 00000000000..2677323ae89 --- /dev/null +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** To implement the functionality of the "quorum write". + * Information about which replicas the inserted part of data appeared on, + * and on how many replicas it should be. + */ +struct ReplicatedMergeTreeBlockEntry +{ + String part_name; + std::optional quorum_status; + + ReplicatedMergeTreeBlockEntry() {} + ReplicatedMergeTreeBlockEntry(const String & str) + { + fromString(str); + } + + void writeText(WriteBuffer & out) const + { + out << part_name << "\n"; + + if (quorum_status) + quorum_status->writeText(out); + } + + void readText(ReadBuffer & in) + { + in >> part_name; + + if (!in.eof()) + { + in >> "\n"; + quorum_status = ReplicatedMergeTreeQuorumStatusEntry(); + quorum_status->readText(in); + } + } + + String toString() const + { + WriteBufferFromOwnString out; + writeText(out); + return out.str(); + } + + void fromString(const String & str) + { + ReadBufferFromString in(str); + readText(in); + } +}; + +} diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 5696a9cf890..f56176f4ff6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -39,12 +40,14 @@ ReplicatedMergeTreeBlockOutputStream::ReplicatedMergeTreeBlockOutputStream( size_t quorum_, size_t quorum_timeout_ms_, size_t max_parts_per_block_, + bool quorum_parallel_, bool deduplicate_) : storage(storage_) , metadata_snapshot(metadata_snapshot_) , quorum(quorum_) , quorum_timeout_ms(quorum_timeout_ms_) , max_parts_per_block(max_parts_per_block_) + , quorum_parallel(quorum_parallel_) , deduplicate(deduplicate_) , log(&Poco::Logger::get(storage.getLogName() + " (Replicated OutputStream)")) { @@ -243,15 +246,21 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( Int64 block_number = 0; String existing_part_name; + ReplicatedMergeTreeBlockEntry block_entry; if (block_number_lock) { is_already_existing_part = false; block_number = block_number_lock->getNumber(); + block_entry.part_name = part->name; /// Set part attributes according to part_number. Prepare an entry for log. part->info.min_block = block_number; part->info.max_block = block_number; + + /// ALEXELEXA + /// somehow need to send this block_if to part node. TODO + part->info.block_id = block_id; part->info.level = 0; part->name = part->getNewName(part->info); @@ -282,10 +291,9 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( */ if (quorum) { - ReplicatedMergeTreeQuorumEntry quorum_entry; - quorum_entry.part_name = part->name; - quorum_entry.required_number_of_replicas = quorum; - quorum_entry.replicas.insert(storage.replica_name); + ReplicatedMergeTreeQuorumStatusEntry status_entry; + status_entry.required_number_of_replicas = quorum; + status_entry.replicas.insert(storage.replica_name); /** At this point, this node will contain information that the current replica received a part. * When other replicas will receive this part (in the usual way, processing the replication log), @@ -294,11 +302,21 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( * which indicates that the quorum has been reached. */ - ops.emplace_back( - zkutil::makeCreateRequest( - quorum_info.status_path, - quorum_entry.toString(), - zkutil::CreateMode::Persistent)); + if (!quorum_parallel) + { + ReplicatedMergeTreeQuorumEntry quorum_entry; + quorum_entry.part_name = part->name; + quorum_entry.status = status_entry; + + ops.emplace_back( + zkutil::makeCreateRequest( + quorum_info.status_path, + quorum_entry.toString(), + zkutil::CreateMode::Persistent)); + + } + else + block_entry.quorum_status = status_entry; /// Make sure that during the insertion time, the replica was not reinitialized or disabled (when the server is finished). ops.emplace_back( @@ -352,7 +370,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( } /// Information about the part. - storage.getCommitPartOps(ops, part, block_id_path); + storage.getCommitPartOps(ops, part, block_id_path, block_entry); MergeTreeData::Transaction transaction(storage); /// If you can not add a part to ZK, we'll remove it back from the working set. bool renamed = false; @@ -466,13 +484,15 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( if (is_already_existing_part) { /// We get duplicate part without fetch - storage.updateQuorum(part->name); + /// ALEXELEXA + /// should reset here something, after thinking in TODO + storage.updateQuorum(part->name, part->info.block_id); } /// We are waiting for quorum to be satisfied. LOG_TRACE(log, "Waiting for quorum"); - String quorum_status_path = storage.zookeeper_path + "/quorum/status"; + String quorum_status_path = quorum_parallel ? storage.zookeeper_path + "/blocks/" + block_id : storage.zookeeper_path + "/quorum/status"; try { @@ -481,15 +501,25 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( zkutil::EventPtr event = std::make_shared(); std::string value; + ReplicatedMergeTreeQuorumEntry quorum_entry; + ReplicatedMergeTreeBlockEntry block_entry; /// `get` instead of `exists` so that `watch` does not leak if the node is no longer there. if (!zookeeper->tryGet(quorum_status_path, value, nullptr, event)) break; - ReplicatedMergeTreeQuorumEntry quorum_entry(value); - /// If the node has time to disappear, and then appear again for the next insert. - if (quorum_entry.part_name != part->name) - break; + if (quorum_parallel) + { + block_entry.fromString(value); + if (block_entry.part_name != part->name) + break; + } + else + { + quorum_entry.fromString(value); + if (quorum_entry.part_name != part->name) + break; + } if (!event->tryWait(quorum_timeout_ms)) throw Exception("Timeout while waiting for quorum", ErrorCodes::TIMEOUT_EXCEEDED); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h index fa3ede20c28..97c094c1128 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h @@ -28,6 +28,7 @@ public: size_t quorum_, size_t quorum_timeout_ms_, size_t max_parts_per_block_, + bool quorum_parallel_, bool deduplicate_); Block getHeader() const override; @@ -64,6 +65,7 @@ private: size_t quorum_timeout_ms; size_t max_parts_per_block; + bool quorum_parallel = false; bool deduplicate = true; bool last_block_is_duplicate = false; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 11f23a5c110..4df47915048 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -70,6 +70,8 @@ void ReplicatedMergeTreeCleanupThread::iterate() if (storage.is_leader) { clearOldLogs(); + /// ALEXELEXA + /// may be just remove it? clearOldBlocks(); clearOldMutations(); } @@ -344,6 +346,8 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() for (auto it = first_outdated_block; it != timed_blocks.end(); ++it) { String path = storage.zookeeper_path + "/blocks/" + it->node; + /// ALEXELEXA + /// should check somehow is it quorum block try_remove_futures.emplace_back(path, zookeeper->asyncTryRemove(path)); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp index 88f4a3ec66f..db52f05c0c9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp @@ -34,11 +34,24 @@ ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromColumnsAndCheck return ReplicatedMergeTreePartHeader(getSipHash(columns.toString()), std::move(checksums)); } +ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromColumnsChecksumsBlockID( + const NamesAndTypesList & columns, + const MergeTreeDataPartChecksums & full_checksums, + const String & block_id_) +{ + MinimalisticDataPartChecksums checksums; + checksums.computeTotalChecksums(full_checksums); + return ReplicatedMergeTreePartHeader(getSipHash(columns.toString()), std::move(checksums), block_id_); +} + void ReplicatedMergeTreePartHeader::read(ReadBuffer & in) { in >> "part header format version: 1\n"; in.readStrict(columns_hash.data(), columns_hash.size()); checksums.deserializeWithoutHeader(in); + + if (!in.eof()) + in >> "block_id: " >> block_id.value() >> "\n"; } ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromString(const String & str) @@ -54,6 +67,8 @@ void ReplicatedMergeTreePartHeader::write(WriteBuffer & out) const writeString("part header format version: 1\n", out); out.write(columns_hash.data(), columns_hash.size()); checksums.serializeWithoutHeader(out); + if (block_id) + out << "block_id " << block_id.value() << "\n"; } String ReplicatedMergeTreePartHeader::toString() const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h index 7bc2b72d2d5..d0b8c11dfb6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -28,6 +29,9 @@ public: static ReplicatedMergeTreePartHeader fromColumnsAndChecksums( const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums); + static ReplicatedMergeTreePartHeader fromColumnsChecksumsBlockID( + const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums, const String & block_id_); + void read(ReadBuffer & in); static ReplicatedMergeTreePartHeader fromString(const String & str); @@ -38,13 +42,15 @@ public: const MinimalisticDataPartChecksums & getChecksums() const { return checksums; } private: - ReplicatedMergeTreePartHeader(std::array columns_hash_, MinimalisticDataPartChecksums checksums_) - : columns_hash(std::move(columns_hash_)), checksums(std::move(checksums_)) + ReplicatedMergeTreePartHeader(std::array columns_hash_, MinimalisticDataPartChecksums checksums_, + std::optional block_id_ = std::nullopt) + : columns_hash(std::move(columns_hash_)), checksums(std::move(checksums_)), block_id(std::move(block_id_)) { } std::array columns_hash; MinimalisticDataPartChecksums checksums; + std::optional block_id; }; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h index ee12cabb5aa..93e117d82af 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -20,8 +21,7 @@ namespace DB struct ReplicatedMergeTreeQuorumEntry { String part_name; - size_t required_number_of_replicas{}; - std::set replicas; + ReplicatedMergeTreeQuorumStatusEntry status; ReplicatedMergeTreeQuorumEntry() {} ReplicatedMergeTreeQuorumEntry(const String & str) @@ -32,31 +32,15 @@ struct ReplicatedMergeTreeQuorumEntry void writeText(WriteBuffer & out) const { out << "version: 1\n" - << "part_name: " << part_name << "\n" - << "required_number_of_replicas: " << required_number_of_replicas << "\n" - << "actual_number_of_replicas: " << replicas.size() << "\n" - << "replicas:\n"; - - for (const auto & replica : replicas) - out << escape << replica << "\n"; + << "part_name: " << part_name << "\n"; + status.writeText(out); } void readText(ReadBuffer & in) { - size_t actual_number_of_replicas = 0; - in >> "version: 1\n" - >> "part_name: " >> part_name >> "\n" - >> "required_number_of_replicas: " >> required_number_of_replicas >> "\n" - >> "actual_number_of_replicas: " >> actual_number_of_replicas >> "\n" - >> "replicas:\n"; - - for (size_t i = 0; i < actual_number_of_replicas; ++i) - { - String replica; - in >> escape >> replica >> "\n"; - replicas.insert(replica); - } + >> "part_name: " >> part_name >> "\n"; + status.readText(in); } String toString() const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h new file mode 100644 index 00000000000..bb3cb83b9a1 --- /dev/null +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** To implement the functionality of the "quorum write". + * Information about which replicas the inserted part of data appeared on, + * and on how many replicas it should be. + */ +struct ReplicatedMergeTreeQuorumStatusEntry +{ + size_t required_number_of_replicas{}; + std::set replicas; + + ReplicatedMergeTreeQuorumStatusEntry() {} + ReplicatedMergeTreeQuorumStatusEntry(const String & str) + { + fromString(str); + } + + void writeText(WriteBuffer & out) const + { + out << "required_number_of_replicas: " << required_number_of_replicas << "\n" + << "actual_number_of_replicas: " << replicas.size() << "\n" + << "replicas:\n"; + + for (const auto & replica : replicas) + out << escape << replica << "\n"; + } + + void readText(ReadBuffer & in) + { + size_t actual_number_of_replicas = 0; + + in >> "required_number_of_replicas: " >> required_number_of_replicas >> "\n" + >> "actual_number_of_replicas: " >> actual_number_of_replicas >> "\n" + >> "replicas:\n"; + + for (size_t i = 0; i < actual_number_of_replicas; ++i) + { + String replica; + in >> escape >> replica >> "\n"; + replicas.insert(replica); + } + } + + String toString() const + { + WriteBufferFromOwnString out; + writeText(out); + return out.str(); + } + + void fromString(const String & str) + { + ReadBufferFromString in(str); + readText(in); + } + + bool isQuorumReached() + { + return required_number_of_replicas <= replicas.size(); + } +}; + +} diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 8424da11896..5a89906dd3f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -227,9 +227,9 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() if (zookeeper->tryGet(storage.zookeeper_path + "/quorum/status", quorum_str)) { ReplicatedMergeTreeQuorumEntry quorum_entry; - quorum_entry.fromString(quorum_str); + quorum_entry.status.fromString(quorum_str); - if (!quorum_entry.replicas.count(storage.replica_name) + if (!quorum_entry.status.replicas.count(storage.replica_name) && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", quorum_entry.part_name); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 65c0c5ac313..1df419b6dac 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1176,6 +1176,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: { ops.emplace_back(zkutil::makeCreateRequest( part_path, local_part_header.toString(), zkutil::CreateMode::Persistent)); + LOG_DEBUG(log, "local_part_header.toString(): {}", local_part_header.toString()); } else { @@ -3028,7 +3029,7 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( /** If a quorum is tracked for a part, update information about it in ZK. */ -void StorageReplicatedMergeTree::updateQuorum(const String & part_name) +void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const std::optional & block_id) { auto zookeeper = getZooKeeper(); @@ -3036,25 +3037,44 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) const String quorum_status_path = zookeeper_path + "/quorum/status"; /// The name of the previous part for which the quorum was reached. const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; - - String value; - Coordination::Stat stat; + const String quorum_parallel_status_path = block_id ? zookeeper_path + "/blocks/" + *block_id : ""; /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. - while (zookeeper->tryGet(quorum_status_path, value, &stat)) + String value; + Coordination::Stat stat; + while (true) { + bool is_parallel = false; + ReplicatedMergeTreeBlockEntry block_entry; ReplicatedMergeTreeQuorumEntry quorum_entry; - quorum_entry.fromString(value); - if (quorum_entry.part_name != part_name) + if (zookeeper->tryGet(quorum_status_path, value, &stat)) + { + quorum_entry.fromString(value); + quorum_entry.status.replicas.insert(replica_name); + } + else if (block_id && zookeeper->tryGet(quorum_parallel_status_path, value, &stat)) + { + block_entry.fromString(value); + // The quorum has already been achieved + if (!block_entry.quorum_status) + break; + + is_parallel = true; + block_entry.quorum_status->replicas.insert(replica_name); + if (block_entry.quorum_status->isQuorumReached()) + block_entry.quorum_status.reset(); + } + else + break; + + if (quorum_entry.part_name != part_name && block_entry.part_name != part_name) { /// The quorum has already been achieved. Moreover, another INSERT with a quorum has already started. break; } - quorum_entry.replicas.insert(replica_name); - - if (quorum_entry.replicas.size() >= quorum_entry.required_number_of_replicas) + if (quorum_entry.status.isQuorumReached()) { /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. @@ -3098,8 +3118,16 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) } else { - /// We update the node, registering there one more replica. - auto code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); + Coordination::Error code; + if (is_parallel) + { + /// We update the node, registering there one more replica. + code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); + } + else { + /// Update parallel quorum. It also might be reached here. + code = zookeeper->trySet(quorum_parallel_status_path, block_entry.toString(), stat.version); + } if (code == Coordination::Error::ZOK) { @@ -3588,6 +3616,7 @@ BlockOutputStreamPtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, *this, metadata_snapshot, query_settings.insert_quorum, query_settings.insert_quorum_timeout.totalMilliseconds(), query_settings.max_partitions_per_insert_block, + query_settings.insert_quorum_parallel, deduplicate); } @@ -4164,7 +4193,7 @@ PartitionCommandsResultInfo StorageReplicatedMergeTree::attachPartition( PartsTemporaryRename renamed_parts(*this, "detached/"); MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts); - ReplicatedMergeTreeBlockOutputStream output(*this, metadata_snapshot, 0, 0, 0, false); /// TODO Allow to use quorum here. + ReplicatedMergeTreeBlockOutputStream output(*this, metadata_snapshot, 0, 0, 0, false, false); /// TODO Allow to use quorum here. for (size_t i = 0; i < loaded_parts.size(); ++i) { String old_name = loaded_parts[i]->name; @@ -5242,6 +5271,8 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( if (result.error == Coordination::Error::ZNONODE) continue; + /// ALEXELEXA + /// should parse it here using another way ReadBufferFromString buf(result.data); Int64 block_num = 0; bool parsed = tryReadIntText(block_num, buf) && buf.eof(); @@ -5661,18 +5692,19 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta void StorageReplicatedMergeTree::getCommitPartOps( Coordination::Requests & ops, MutableDataPartPtr & part, - const String & block_id_path) const + const String & block_id_path, + ReplicatedMergeTreeBlockEntry block_entry) const { - const String & part_name = part->name; const auto storage_settings_ptr = getSettings(); if (!block_id_path.empty()) { /// Make final duplicate check and commit block_id + block_entry.part_name = part->name; ops.emplace_back( zkutil::makeCreateRequest( block_id_path, - part_name, /// We will be able to know original part number for duplicate blocks, if we want. + block_entry.toString(), /// We will be able to know original part number for duplicate blocks, if we want. zkutil::CreateMode::Persistent)); } @@ -5683,6 +5715,8 @@ void StorageReplicatedMergeTree::getCommitPartOps( replica_path + "/parts/" + part->name, ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), zkutil::CreateMode::Persistent)); + LOG_DEBUG(log, "ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(): {}", + ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString()); } else { diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 2bc9265331d..daf8b712a6c 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -350,7 +351,8 @@ private: bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; - void getCommitPartOps(Coordination::Requests & ops, MutableDataPartPtr & part, const String & block_id_path = "") const; + void getCommitPartOps(Coordination::Requests & ops, MutableDataPartPtr & part, const String & block_id_path = "", + ReplicatedMergeTreeBlockEntry block_entry = ReplicatedMergeTreeBlockEntry()) const; /// Adds actions to `ops` that remove a part from ZooKeeper. /// Set has_children to true for "old-style" parts (those with /columns and /checksums child znodes). @@ -492,7 +494,7 @@ private: /// With the quorum being tracked, add a replica to the quorum for the part. - void updateQuorum(const String & part_name); + void updateQuorum(const String & part_name, const std::optional & block_id = std::nullopt); /// Deletes info from quorum/last_part node for particular partition_id. void cleanLastPartNode(const String & partition_id); From 8a707b1bb42013c3b83d225289c3937098520906 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 30 Sep 2020 21:40:07 -0400 Subject: [PATCH 080/441] Adding `paralle login of rbac users` test. --- .../tests/authentications.py | 105 ++++++++++++------ .../external_user_directory/tests/common.py | 23 +++- 2 files changed, 89 insertions(+), 39 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 55290d1ada9..cdace3007fa 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -52,57 +52,92 @@ def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) +def login_with_valid_username_and_password(users, i, iterations=10): + """Login with valid username and password. + """ + with When(f"valid users try to login #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + +def login_with_valid_username_and_invalid_password(users, i, iterations=10): + """Login with valid username and invalid password. + """ + with When(f"users try to login with valid username and invalid password #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], + password=(random_user["userpassword"] + randomword(1)), + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + +def login_with_invalid_username_and_valid_password(users, i, iterations=10): + """Login with invalid username and valid password. + """ + with When(f"users try to login with invalid username and valid password #{i}"): + for i in range(iterations): + random_user = dict(users[random.randint(0, len(users)-1)]) + random_user["cn"] += randomword(1) + login_and_execute_query(username=random_user["cn"], + password=random_user["userpassword"], + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") ) def parallel_login(self, server, user_count=10, timeout=200): - """Check that login of valid and invalid LDAP authenticated users works in parallel.""" + """Check that login of valid and invalid LDAP authenticated users works in parallel. + """ self.context.ldap_node = self.context.cluster.node(server) user = None users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] with ldap_users(*users): - def login_with_valid_username_and_password(users, i, iterations=10): - with When(f"valid users try to login #{i}"): - for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + tasks = [] + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - def login_with_valid_username_and_invalid_password(users, i, iterations=10): - with When(f"users try to login with valid username and invalid password #{i}"): - for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], - password=(random_user["userpassword"] + randomword(1)), - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + finally: + with Then("it should work"): + join(tasks, timeout) - def login_with_invalid_username_and_valid_password(users, i, iterations=10): - with When(f"users try to login with invalid username and valid password #{i}"): - for i in range(iterations): - random_user = dict(users[random.randint(0, len(users)-1)]) - random_user["cn"] += randomword(1) - login_and_execute_query(username=random_user["cn"], - password=random_user["userpassword"], - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly("1.0") +) +def parallel_login_of_rbac_users(self, server, user_count=10, timeout=200): + """Check that login of only valid and invalid local users created using RBAC + works in parallel when server configuration includes LDAP external user directory. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None - with When("I login in parallel"): - p = Pool(15) - tasks = [] - for i in range(5): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with Then("it should work"): - for task in tasks: - task.get(timeout=timeout) + with rbac_users(*users): + tasks = [] + + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py index 4c7877035c8..57e5cc34a94 100644 --- a/tests/testflows/ldap/external_user_directory/tests/common.py +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -11,6 +11,21 @@ from ldap.authentication.tests.common import ldap_user, ldap_users, add_user_to_ from ldap.authentication.tests.common import change_user_password_in_ldap, change_user_cn_in_ldap from ldap.authentication.tests.common import randomword +def join(tasks, timeout): + """Join async tasks by waiting for their completion. + """ + task_exc = None + + for task in tasks: + try: + task.get(timeout=timeout) + except Exception as exc: + if task_exc is None: + task_exc = exc + + if task_exc is not None: + raise task_exc + @contextmanager def table(name, create_statement, on_cluster=False): node = current().context.node @@ -31,14 +46,14 @@ def rbac_users(*users): try: with Given("I have local users"): for user in users: - with By(f"creating user {user}"): - node.query(f"CREATE USER OR REPLACE {user} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user}'") + with By(f"creating user {user['cn']}", format_name=False): + node.query(f"CREATE USER OR REPLACE {user['cn']} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user['userpassword']}'") yield users finally: with Finally("I drop local users"): for user in users: - with By(f"dropping user {user}", flags=TE): - node.query(f"DROP USER IF EXISTS {user}") + with By(f"dropping user {user['cn']}", flags=TE, format_name=False): + node.query(f"DROP USER IF EXISTS {user['cn']}") @contextmanager def rbac_roles(*roles): From 077ee81177166b97fdd1fa9acabf0b764a1301e2 Mon Sep 17 00:00:00 2001 From: filipe Date: Wed, 30 Sep 2020 23:55:46 -0300 Subject: [PATCH 081/441] add an SQL function formatReadableTimeDelta to format time delta --- .../functions/other-functions.md | 20 +++ src/Functions/FunctionsFormatting.cpp | 1 + src/Functions/FunctionsFormatting.h | 151 ++++++++++++++++++ .../queries/0_stateless/00534_filimonov.data | 1 + .../01511_format_readable_timedelta.reference | 30 ++++ .../01511_format_readable_timedelta.sql | 6 + 6 files changed, 209 insertions(+) create mode 100755 tests/queries/0_stateless/01511_format_readable_timedelta.reference create mode 100755 tests/queries/0_stateless/01511_format_readable_timedelta.sql diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 1c059e9f97b..b69a0693ed5 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -538,6 +538,26 @@ SELECT └────────────────┴───────────────────┘ ``` +## formatReadableTimeDelta(x) {#formatreadabletimedeltax} + +Accepts the time delta in seconds. Returns a time delta with (year, month, day, hour, minute, second) as a string. + +Example: + +``` sql +SELECT + arrayJoin([100, 12345, 432546534]) AS number, + formatReadableTimeDelta(number) AS time_delta +``` + +``` text +┌─────number─┬─time_delta ─────────────────────────────────────────────────────┐ +│ 100 │ 1 minute and 40 seconds │ +│ 12345 │ 3 hours, 25 minutes and 45 seconds │ +│ 432546534 │ 13 years, 8 months, 17 days, 7 hours, 48 minutes and 54 seconds │ +└────────────┴─────────────────────────────────────────────────────────────────┘ +``` + ## least(a, b) {#leasta-b} Returns the smallest value from a and b. diff --git a/src/Functions/FunctionsFormatting.cpp b/src/Functions/FunctionsFormatting.cpp index 1187749aa2d..c7037fdd1a3 100644 --- a/src/Functions/FunctionsFormatting.cpp +++ b/src/Functions/FunctionsFormatting.cpp @@ -10,6 +10,7 @@ void registerFunctionsFormatting(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index da22babec38..3c48772d840 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -278,4 +278,155 @@ private: } }; + +class FunctionFormatReadableTimeDelta : public IFunction +{ +public: + static constexpr auto name = "formatReadableTimeDelta"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 1; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const IDataType & type = *arguments[0]; + + if (!isNativeNumber(type)) + throw Exception("Cannot format " + type.getName() + " as time delta", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + bool useDefaultImplementationForConstants() const override { return true; } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + { + if (!(executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result) + || executeType(block, arguments, result))) + throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() + + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + +private: + void formatReadableTimeDelta(double value, DB::WriteBuffer & out) const + { + // 60 SECONDS = 1 MINUTE + // 3600 SECONDS = 1 HOUR + // 86400 SECONDS = 1 DAY + // 30.5 DAYS = 1 MONTH + // 365 DAYS = 1 YEAR + + char sig = value<0?-1:1; + value *= sig; + + long long int date = value / 86400; + double time = value - date * 86400; + + long long int hours = time / 3600; + long long int minutes = (time - hours * 3600) / 60; + double seconds = time - hours * 3600 - minutes * 60; + + long long int years = date / 365; + long long int months = (date - years * 365) / 30.5; + long long int days = date - years * 365 - months * 30.5; + + std::vector parts; + + if (years) + { + parts.push_back(std::to_string(years) + (years==1?" year":" years")); + } + if (months) + { + parts.push_back(std::to_string(months) + (months==1?"month":" months")); + } + if (days) + { + parts.push_back(std::to_string(days) + (days==1?" day":" days")); + } + // Test overflow 2^64 + // If overflow happens then hide the time + if (time < 9223372036854775808.0) + { + if (hours) + { + parts.push_back(std::to_string(hours) + (hours==1?" hour":" hours")); + } + if (minutes) + { + parts.push_back(std::to_string(minutes) + (minutes==1?" minute":" minutes")); + } + if (seconds) + { + std::string seconds_str = std::to_string(seconds); + seconds_str.erase(seconds_str.find_last_not_of('0') + 1, std::string::npos); + seconds_str.erase(seconds_str.find_last_not_of('.') + 1, std::string::npos); + parts.push_back(seconds_str + (seconds==1?" second":" seconds")); + } + } + + + String str_value; + for(size_t i=0; i + bool executeType(Block & block, const ColumnNumbers & arguments, size_t result) const + { + if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) + { + auto col_to = ColumnString::create(); + + const typename ColumnVector::Container & vec_from = col_from->getData(); + ColumnString::Chars & data_to = col_to->getChars(); + ColumnString::Offsets & offsets_to = col_to->getOffsets(); + size_t size = vec_from.size(); + data_to.resize(size * 2); + offsets_to.resize(size); + + WriteBufferFromVector buf_to(data_to); + + for (size_t i = 0; i < size; ++i) + { + formatReadableTimeDelta(static_cast(vec_from[i]), buf_to); + writeChar(0, buf_to); + offsets_to[i] = buf_to.count(); + } + + buf_to.finalize(); + block.getByPosition(result).column = std::move(col_to); + return true; + } + + return false; + } +}; + } diff --git a/tests/queries/0_stateless/00534_filimonov.data b/tests/queries/0_stateless/00534_filimonov.data index 393ff5b7155..ee4d2caa029 100644 --- a/tests/queries/0_stateless/00534_filimonov.data +++ b/tests/queries/0_stateless/00534_filimonov.data @@ -175,6 +175,7 @@ SELECT protocol(NULL); SELECT toInt16OrZero(NULL); SELECT formatReadableSize(NULL); SELECT formatReadableQuantity(NULL); +SELECT formatReadableTimeDelta(NULL); SELECT concatAssumeInjective(NULL); SELECT toString(NULL); SELECT MACStringToNum(NULL); diff --git a/tests/queries/0_stateless/01511_format_readable_timedelta.reference b/tests/queries/0_stateless/01511_format_readable_timedelta.reference new file mode 100755 index 00000000000..04a41c8e08a --- /dev/null +++ b/tests/queries/0_stateless/01511_format_readable_timedelta.reference @@ -0,0 +1,30 @@ +1 second 1 second +2 seconds 2 seconds +7 seconds 7 seconds +20 seconds 20 seconds +54 seconds 54 seconds +2 minutes and 28 seconds 2 minutes and 28 seconds +6 minutes and 43 seconds 6 minutes and 43 seconds +18 minutes and 16 seconds 18 minutes and 16 seconds +49 minutes and 40 seconds 49 minutes and 40 seconds +2 hours, 15 minutes and 3 seconds 2 hours, 15 minutes and 3 seconds +6 hours, 7 minutes and 6 seconds 6 hours, 7 minutes and 6 seconds +16 hours, 37 minutes and 54 seconds 16 hours, 37 minutes and 54 seconds +1 day, 21 hours, 12 minutes and 34 seconds 1 day, 21 hours, 12 minutes and 34 seconds +5 days, 2 hours, 53 minutes and 33 seconds 5 days, 2 hours, 53 minutes and 33 seconds +13 days, 22 hours, 3 minutes and 24 seconds 13 days, 22 hours, 3 minutes and 24 seconds +1month, 6 days, 20 hours, 3 minutes and 37 seconds 1month, 6 days, 20 hours, 3 minutes and 37 seconds +3 months, 10 days, 20 hours, 21 minutes and 50 seconds 3 months, 10 days, 20 hours, 21 minutes and 50 seconds +9 months, 4 days, 13 hours, 42 minutes and 32 seconds 9 months, 4 days, 13 hours, 42 minutes and 32 seconds +2 years, 29 days, 22 hours, 52 minutes and 49 seconds 2 years, 29 days, 22 hours, 52 minutes and 49 seconds +5 years, 7 months, 26 days, 18 hours and 25 minutes 5 years, 7 months, 26 days, 18 hours and 25 minutes +15 years, 4 months, 18 days, 8 hours, 6 minutes and 35 seconds 15 years, 4 months, 18 days, 8 hours, 6 minutes and 35 seconds +41 years, 9 months, 24 days, 1 hour, 42 minutes and 14 seconds 41 years, 9 months, 24 days, 1 hour, 42 minutes and 14 seconds +113 years, 8 months, 3 days, 1 hour, 7 minutes and 26 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +309 years, 2 days, 1 hour, 50 minutes and 46 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +839 years, 11 months, 16 days, 1 hour, 28 minutes and 49 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +2283 years, 3 months, 3 days, 55 minutes and 37 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +6206 years, 6 months, 15 days, 23 hours, 57 minutes and 8 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +16871 years, 1month, 19 days, 17 hours, 56 minutes and 42 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +45860 years, 6 months, 3 days, 9 hours, 24 minutes and 52 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +124661 years, 9 months, 14 days, 8 hours, 45 minutes and 42 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds diff --git a/tests/queries/0_stateless/01511_format_readable_timedelta.sql b/tests/queries/0_stateless/01511_format_readable_timedelta.sql new file mode 100755 index 00000000000..a71742d540c --- /dev/null +++ b/tests/queries/0_stateless/01511_format_readable_timedelta.sql @@ -0,0 +1,6 @@ +WITH round(exp(number), 6) AS x, toUInt64(x) AS y, toInt32(x) AS z +SELECT + formatReadableTimeDelta(y), + formatReadableTimeDelta(z) +FROM system.numbers +LIMIT 30; From 80c334a5ace304b17b39a32a128911188ea6c23e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 1 Oct 2020 07:21:38 -0400 Subject: [PATCH 082/441] Fixing user_lookup_priority test. --- .../ldap/external_user_directory/tests/authentications.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index cdace3007fa..971243b1d48 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -127,7 +127,6 @@ def parallel_login_of_rbac_users(self, server, user_count=10, timeout=200): with rbac_users(*users): tasks = [] - try: with When("I login in parallel"): p = Pool(15) @@ -509,7 +508,7 @@ def user_lookup_priority(self, server): } with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): - with rbac_users("local"): + with rbac_users({"cn": "local", "userpassword": "local"}): with When("I try to login as 'default' user which is also defined in users.xml it should fail"): login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) From ab2c37cead84d29815e1620a7414b98600527cdb Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 00:31:14 +0400 Subject: [PATCH 083/441] Serialize all calls to ldap lib --- src/Access/LDAPClient.cpp | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index a85e96ab86c..d6580b89c68 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -2,6 +2,8 @@ #include #include +#include + #include #include @@ -27,16 +29,13 @@ LDAPClient::~LDAPClient() closeConnection(); } -void LDAPClient::openConnection() -{ - const bool graceful_bind_failure = false; - diag(openConnection(graceful_bind_failure)); -} - #if USE_LDAP namespace { + + std::recursive_mutex ldap_global_mutex; + auto escapeForLDAP(const String & src) { String dest; @@ -63,10 +62,13 @@ namespace return dest; } + } void LDAPClient::diag(const int rc) { + std::scoped_lock lock(ldap_global_mutex); + if (rc != LDAP_SUCCESS) { String text; @@ -100,8 +102,18 @@ void LDAPClient::diag(const int rc) } } +void LDAPClient::openConnection() +{ + std::scoped_lock lock(ldap_global_mutex); + + const bool graceful_bind_failure = false; + diag(openConnection(graceful_bind_failure)); +} + int LDAPClient::openConnection(const bool graceful_bind_failure) { + std::scoped_lock lock(ldap_global_mutex); + closeConnection(); { @@ -258,6 +270,8 @@ int LDAPClient::openConnection(const bool graceful_bind_failure) void LDAPClient::closeConnection() noexcept { + std::scoped_lock lock(ldap_global_mutex); + if (!handle) return; @@ -267,6 +281,8 @@ void LDAPClient::closeConnection() noexcept bool LDAPSimpleAuthClient::check() { + std::scoped_lock lock(ldap_global_mutex); + if (params.user.empty()) throw Exception("LDAP authentication of a user with an empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); @@ -312,6 +328,11 @@ void LDAPClient::diag(const int) throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } +void LDAPClient::openConnection() +{ + throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); +} + int LDAPClient::openConnection(const bool) { throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); From 68ccd59a7495f5e1592269cd6d12f31cb1de2f93 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 00:32:13 +0400 Subject: [PATCH 084/441] Synch with internal memory_storage Fix exception message --- src/Access/LDAPAccessStorage.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index ae61d912ebe..38922eeac55 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -69,6 +69,7 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) { + std::scoped_lock lock(mutex); auto role_ptr = typeid_cast>(entity); if (role_ptr) { @@ -123,6 +124,7 @@ const char * LDAPAccessStorage::getStorageType() const String LDAPAccessStorage::getStorageParamsJSON() const { + std::scoped_lock lock(mutex); Poco::JSON::Object params_json; params_json.set("server", ldap_server); @@ -137,30 +139,35 @@ String LDAPAccessStorage::getStorageParamsJSON() const std::optional LDAPAccessStorage::findImpl(EntityType type, const String & name) const { + std::scoped_lock lock(mutex); return memory_storage.find(type, name); } std::vector LDAPAccessStorage::findAllImpl(EntityType type) const { + std::scoped_lock lock(mutex); return memory_storage.findAll(type); } bool LDAPAccessStorage::existsImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.exists(id); } AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.read(id); } String LDAPAccessStorage::readNameImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.readName(id); } @@ -179,6 +186,7 @@ UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) void LDAPAccessStorage::removeImpl(const UUID & id) { + std::scoped_lock lock(mutex); auto entity = read(id); throwReadonlyCannotRemove(entity->getType(), entity->getName()); } @@ -186,6 +194,7 @@ void LDAPAccessStorage::removeImpl(const UUID & id) void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) { + std::scoped_lock lock(mutex); auto entity = read(id); throwReadonlyCannotUpdate(entity->getType(), entity->getName()); } @@ -193,24 +202,28 @@ void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { + std::scoped_lock lock(mutex); return memory_storage.subscribeForChanges(id, handler); } ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { + std::scoped_lock lock(mutex); return memory_storage.subscribeForChanges(type, handler); } bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.hasSubscription(id); } bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const { + std::scoped_lock lock(mutex); return memory_storage.hasSubscription(type); } @@ -261,7 +274,7 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw } catch (...) { - tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + access_control_manager->getStorageName() + "'"); + tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + getStorageName() + "'"); } throwCannotAuthenticate(user_name); } From 8ee10a179037b08cde164a070d1f154386c13e64 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 00:49:51 +0400 Subject: [PATCH 085/441] Merge artefact --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index 8aea4e168e7..a6570b863cf 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 8aea4e168e78f3eb9828080740fc8cb73d53bf79 +Subproject commit a6570b863cf76c9699580ba51c7827d5bffaac43 From 92ec9e871684c4403d4d7c3888b59f31f1c06cd1 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 01:40:23 +0400 Subject: [PATCH 086/441] Merge artefact --- contrib/cyrus-sasl | 2 +- contrib/poco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/cyrus-sasl b/contrib/cyrus-sasl index 6054630889f..9995bf9d8e1 160000 --- a/contrib/cyrus-sasl +++ b/contrib/cyrus-sasl @@ -1 +1 @@ -Subproject commit 6054630889fd1cd8d0659573d69badcee1e23a00 +Subproject commit 9995bf9d8e14f58934d9313ac64f13780d6dd3c9 diff --git a/contrib/poco b/contrib/poco index 297fc905e16..757d947235b 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 297fc905e166392156f83b96aaa5f44e8a6a35c4 +Subproject commit 757d947235b307675cff964f29b19d388140a9eb From eb1b3e39e7edf834758ae9713da26aa1b52ee901 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 2 Oct 2020 17:40:24 -0400 Subject: [PATCH 087/441] Adding additional parallel login tests to the external user directory feature. --- .../ldap/authentication/tests/common.py | 85 ++++--- .../tests/authentications.py | 209 +++++++++++++++++- 2 files changed, 254 insertions(+), 40 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 9e8e3afcd1d..979711652db 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -54,6 +54,48 @@ def add_config(config, timeout=20, restart=False): :param config: configuration file description :param timeout: timeout, default: 20 sec """ + def check_preprocessed_config_is_updated(): + """Check that preprocessed config is updated. + """ + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + def wait_for_config_to_be_loaded(): + """Wait for config to be loaded. + """ + if restart: + with When("I close terminal to the node to be restarted"): + bash.close() + + with And("I get the current log size"): + logsize = \ + node.command("ls -s --block-size=1 /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ + 0].strip() + + with And("I restart ClickHouse to apply the config changes"): + node.restart(safe=False) + + with Then("I tail the log file from using previous log size as the offset"): + bash.prompt = bash.__class__.prompt + bash.open() + bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + + with Then("I wait for config reload message in the log file"): + if restart: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", + timeout=timeout) + else: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", + timeout=timeout) + node = current().context.node try: with Given(f"{config.name}"): @@ -70,29 +112,10 @@ def add_config(config, timeout=20, restart=False): node.command(command, steps=False, exitcode=0) with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break - time.sleep(1) - assert exitcode == 0, error() + check_preprocessed_config_is_updated() - if restart: - bash.close() - logsize = node.command("ls -s --block-size=1 /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[0].strip() - with When("I restart ClickHouse to apply the config changes"): - node.restart(safe=False) - bash.prompt = bash.__class__.prompt - bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") - - with When("I wait for config to be loaded"): - if restart: - bash.expect(f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", timeout=timeout) - else: - bash.expect(f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() yield finally: with Finally(f"I remove {config.name}"): @@ -103,20 +126,11 @@ def add_config(config, timeout=20, restart=False): with By("removing the config file", description=config.path): node.command(f"rm -rf {config.path}", exitcode=0) - with Then(f"{config.preprocessed_name} should be updated"): - started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep '{config.uid}'{' > /dev/null' if not settings.debug else ''}" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 1: - break - time.sleep(1) - assert exitcode == 1, error() - - with When("I wait for config to be loaded"): - started = time.time() - bash.expect(f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + check_preprocessed_config_is_updated() + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml"): """Create LDAP servers configuration content. @@ -288,6 +302,7 @@ def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=No } lines = [] + for key, value in list(user.items()): if key.startswith("_"): continue diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 971243b1d48..cf8620b3dee 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -96,12 +96,17 @@ def parallel_login(self, server, user_count=10, timeout=200): self.context.ldap_node = self.context.cluster.node(server) user = None - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + with Given("a group of LDAP users"): + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] with ldap_users(*users): tasks = [] try: - with When("I login in parallel"): + with When("users try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): p = Pool(15) for i in range(25): tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) @@ -112,11 +117,207 @@ def parallel_login(self, server, user_count=10, timeout=200): with Then("it should work"): join(tasks, timeout) +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_the_same_user(self, server, timeout=200): + """Check that valid and invalid logins of the same + LDAP authenticated user works in parallel. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + with Given("only one LDAP user"): + users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] + + with ldap_users(*users): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + +@TestScenario +def login_after_ldap_external_user_directory_is_removed(self, server): + """Check that ClickHouse stops authenticating LDAP users + after LDAP external user directory is removed. + """ + with When("I attempt to login after LDAP external user directory is added"): + with ldap_external_user_directory(server="openldap2", roles=[], restart=True): + login_and_execute_query(username="user2", password="user2") + + with When("I attempt to login after LDAP external user directory is removed"): + exitcode = 4 + message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=200): + """Check that valid and invalid logins of the same + user defined in multiple LDAP external user directories + works in parallel. + """ + with Given("I have two LDAP servers"): + entries = [ + (["openldap1"], []), + (["openldap2"], []) + ] + + with Given("I define only one LDAP user"): + users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] + + with And("I create config file to define LDAP external user directory for each LDAP server"): + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_users(*users, node=self.context.cluster.node("openldap1")): + with ldap_users(*users, node=self.context.cluster.node("openldap2")): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_multiple_servers(self, server, user_count=10, timeout=200): + """Check that login of valid and invalid LDAP authenticated users works in parallel + using multiple LDAP external user directories. + """ + with Given("I have two LDAP servers"): + entries = [ + (["openldap1"], []), + (["openldap2"], []) + ] + + with And("I define a group of users to be created on each LDAP server"): + user_groups = { + "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], + "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + } + + with And("I have a list of checks that I want to run for each user group"): + checks = [ + login_with_valid_username_and_password, + login_with_valid_username_and_invalid_password, + login_with_invalid_username_and_valid_password + ] + + with And("I create config file to define LDAP external user directory for each LDAP server"): + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): + with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + tasks = [] + + try: + with When("users in each group try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + for users in user_groups.values(): + for check in checks: + tasks.append(p.apply_async(check, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_rbac_and_multiple_servers(self, server, user_count=10, timeout=200): + """Check that login of valid and invalid users works in parallel + using local users defined using RBAC and LDAP users authenticated using + multiple LDAP external user directories. + """ + with Given("I have two LDAP servers"): + entries = [ + (["openldap1"], []), + (["openldap2"], []) + ] + + with And("I define a group of users to be created on each LDAP server"): + user_groups = { + "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], + "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], + "local_users": [{"cn": f"local_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + } + + with And("I have a list of checks that I want to run for each user group"): + checks = [ + login_with_valid_username_and_password, + login_with_valid_username_and_invalid_password, + login_with_invalid_username_and_valid_password + ] + + with And("I create config file to define LDAP external user directory for each LDAP server"): + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): + with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + with rbac_users(*user_groups["local_users"]): + tasks = [] + + try: + with When("users in each group try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + for users in user_groups.values(): + for check in checks: + tasks.append(p.apply_async(check, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly("1.0") ) -def parallel_login_of_rbac_users(self, server, user_count=10, timeout=200): +def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200): """Check that login of only valid and invalid local users created using RBAC works in parallel when server configuration includes LDAP external user directory. """ @@ -362,8 +563,6 @@ def valid_username_and_password_invalid_server(self, server=None): password but for a different server.""" self.context.ldap_node = self.context.cluster.node("openldap1") - user = {"username": "user2", "userpassword": "user2", "server": "openldap1"} - exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" From 82475088f9779b17e28498376b9f02d2e2103c79 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 17:31:02 +0400 Subject: [PATCH 088/441] Fix "user has been dropped" issue --- src/Access/LDAPAccessStorage.cpp | 19 ++++--------------- src/Access/LDAPClient.cpp | 2 +- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 38922eeac55..0142b98f1bb 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -29,11 +29,11 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix) { + std::scoped_lock lock(mutex); + // TODO: switch to passing config as a ConfigurationView and remove this extra prefix once a version of Poco with proper implementation is available. const String prefix_str = (prefix.empty() ? "" : prefix + "."); - std::scoped_lock lock(mutex); - const bool has_server = config.has(prefix_str + "server"); const bool has_roles = config.has(prefix_str + "roles"); @@ -235,20 +235,9 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw auto id = memory_storage.find(user_name); if (id) { - // We try to re-authenticate the existing user, and if not successful, we will remove it, since that would mean - // something changed and the user we authenticated previously cannot be authenticated anymore. auto user = memory_storage.tryRead(*id); - try - { - if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) - return *id; - } - catch (...) - { - memory_storage.remove(*id); - throw; - } - memory_storage.remove(*id); + if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) + return *id; } else { diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index d6580b89c68..d3231f62f3b 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -284,7 +284,7 @@ bool LDAPSimpleAuthClient::check() std::scoped_lock lock(ldap_global_mutex); if (params.user.empty()) - throw Exception("LDAP authentication of a user with an empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); + throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); if (params.password.empty()) return false; // Silently reject authentication attempt if the password is empty as if it didn't match. From 146f973437b0a9d62c047efee1fd583f823501e8 Mon Sep 17 00:00:00 2001 From: filipe Date: Sat, 3 Oct 2020 19:38:51 -0300 Subject: [PATCH 089/441] add maximum_unit arg to formatReadableTimeDelta function --- .../functions/other-functions.md | 33 ++++- src/Functions/FunctionsFormatting.h | 129 +++++++++++++----- .../01511_format_readable_timedelta.reference | 60 ++++---- .../01511_format_readable_timedelta.sql | 25 +++- 4 files changed, 172 insertions(+), 75 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index b69a0693ed5..f70f0841421 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -538,26 +538,51 @@ SELECT └────────────────┴───────────────────┘ ``` -## formatReadableTimeDelta(x) {#formatreadabletimedeltax} +## formatReadableTimeDelta {#formatreadabletimedelta} Accepts the time delta in seconds. Returns a time delta with (year, month, day, hour, minute, second) as a string. +**Syntax** + +``` sql +formatReadableTimeDelta(column[, maximum_unit]) +``` + +**Parameters** + +- `column` — A column with numeric time delta. +- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. + Example: ``` sql SELECT - arrayJoin([100, 12345, 432546534]) AS number, - formatReadableTimeDelta(number) AS time_delta + arrayJoin([100, 12345, 432546534]) AS elapsed, + formatReadableTimeDelta(elapsed) AS time_delta ``` ``` text -┌─────number─┬─time_delta ─────────────────────────────────────────────────────┐ +┌────elapsed─┬─time_delta ─────────────────────────────────────────────────────┐ │ 100 │ 1 minute and 40 seconds │ │ 12345 │ 3 hours, 25 minutes and 45 seconds │ │ 432546534 │ 13 years, 8 months, 17 days, 7 hours, 48 minutes and 54 seconds │ └────────────┴─────────────────────────────────────────────────────────────────┘ ``` +``` sql +SELECT + arrayJoin([100, 12345, 432546534]) AS elapsed, + formatReadableTimeDelta(elapsed, 'minutes') AS time_delta +``` + +``` text +┌────elapsed─┬─time_delta ─────────────────────────────────────────────────────┐ +│ 100 │ 1 minute and 40 seconds │ +│ 12345 │ 205 minutes and 45 seconds │ +│ 432546534 │ 7209108 minutes and 54 seconds │ +└────────────┴─────────────────────────────────────────────────────────────────┘ +``` + ## least(a, b) {#leasta-b} Returns the smallest value from a and b. diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index 3c48772d840..ab5166bac46 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -18,6 +18,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; } @@ -285,23 +286,44 @@ public: static constexpr auto name = "formatReadableTimeDelta"; static FunctionPtr create(const Context &) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 1; } + bool isVariadic() const override { return true; } + + size_t getNumberOfArguments() const override { return 0; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { + if (arguments.size() < 1) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be at least 1.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (arguments.size() > 2) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(arguments.size()) + + ", should be at most 2.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + const IDataType & type = *arguments[0]; if (!isNativeNumber(type)) throw Exception("Cannot format " + type.getName() + " as time delta", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (arguments.size() == 2) + { + const auto * maximum_unit_arg = arguments[1].get(); + if (!isStringOrFixedString(maximum_unit_arg)) + throw Exception("Illegal type " + maximum_unit_arg->getName() + " of argument maximum_unit of function " + + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + return std::make_shared(); } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + bool useDefaultImplementationForConstants() const override { return true; } void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override @@ -322,46 +344,73 @@ public: } private: - void formatReadableTimeDelta(double value, DB::WriteBuffer & out) const + + void formatReadableTimeDelta(double value, String maximum_unit, DB::WriteBuffer & out) const { - // 60 SECONDS = 1 MINUTE - // 3600 SECONDS = 1 HOUR - // 86400 SECONDS = 1 DAY - // 30.5 DAYS = 1 MONTH - // 365 DAYS = 1 YEAR + // 60 SECONDS = 1 MINUTE + // 3600 SECONDS = 1 HOUR + // 86400 SECONDS = 1 DAY + // 2635200 SECONDS = 30.5 DAYS = 1 MONTH + // 31536000 SECONDS = 365 DAYS = 1 YEAR char sig = value<0?-1:1; + int maximum_unit_int = 6; + + if (maximum_unit == "seconds") + maximum_unit_int = 1; + else if (maximum_unit == "minutes") + maximum_unit_int = 2; + else if (maximum_unit == "hours") + maximum_unit_int = 3; + else if (maximum_unit == "days") + maximum_unit_int = 4; + else if (maximum_unit == "months") + maximum_unit_int = 5; + else if (maximum_unit == "years") + maximum_unit_int = 6; + value *= sig; - long long int date = value / 86400; - double time = value - date * 86400; + double aux = 0; - long long int hours = time / 3600; - long long int minutes = (time - hours * 3600) / 60; - double seconds = time - hours * 3600 - minutes * 60; - - long long int years = date / 365; - long long int months = (date - years * 365) / 30.5; - long long int days = date - years * 365 - months * 30.5; + long long int years = maximum_unit_int < 6 ? 0 : value / 31536000; + aux += years * 31536000; + long long int months = maximum_unit_int < 5 ? 0 : (value - aux) / 2635200; + aux += months * 2635200; + long long int days = maximum_unit_int < 4 ? 0 : (value - aux) / 86400; + aux += days * 86400; + long long int hours = maximum_unit_int < 3 ? 0 : (value - aux) / 3600; + aux += hours * 3600; + long long int minutes = maximum_unit_int < 2 ? 0 : (value - aux) / 60; + aux += minutes * 60; + double seconds = maximum_unit_int < 1 ? 0 : value - aux; std::vector parts; - if (years) + /* If value is bigger than 2**64 (292471208677 years) overflow happens + To prevent wrong results the function shows only the year + and maximum_unit is ignored + */ + if (value > 9223372036854775808.0) { - parts.push_back(std::to_string(years) + (years==1?" year":" years")); + std::string years_str = std::to_string(value/31536000.0); + years_str.erase(years_str.find('.'), std::string::npos); + parts.push_back(years_str + " years"); } - if (months) - { - parts.push_back(std::to_string(months) + (months==1?"month":" months")); - } - if (days) - { - parts.push_back(std::to_string(days) + (days==1?" day":" days")); - } - // Test overflow 2^64 - // If overflow happens then hide the time - if (time < 9223372036854775808.0) + else { + if (years) + { + parts.push_back(std::to_string(years) + (years==1?" year":" years")); + } + if (months) + { + parts.push_back(std::to_string(months) + (months==1?"month":" months")); + } + if (days) + { + parts.push_back(std::to_string(days) + (days==1?" day":" days")); + } if (hours) { parts.push_back(std::to_string(hours) + (hours==1?" hour":" hours")); @@ -379,7 +428,6 @@ private: } } - String str_value; for(size_t i=0; i bool executeType(Block & block, const ColumnNumbers & arguments, size_t result) const { + String maximum_unit = ""; + if (arguments.size() == 2) + { + const ColumnPtr & maximum_unit_column = block.getByPosition(arguments[1]).column; + if (const ColumnConst * maximum_unit_const_col = checkAndGetColumnConstStringOrFixedString(maximum_unit_column.get())) + maximum_unit = maximum_unit_const_col->getValue(); + else + throw Exception( + "Illegal column " + maximum_unit_const_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + } + if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) { auto col_to = ColumnString::create(); @@ -415,7 +474,7 @@ private: for (size_t i = 0; i < size; ++i) { - formatReadableTimeDelta(static_cast(vec_from[i]), buf_to); + formatReadableTimeDelta(static_cast(vec_from[i]), maximum_unit, buf_to); writeChar(0, buf_to); offsets_to[i] = buf_to.count(); } diff --git a/tests/queries/0_stateless/01511_format_readable_timedelta.reference b/tests/queries/0_stateless/01511_format_readable_timedelta.reference index 04a41c8e08a..5e67a51a190 100755 --- a/tests/queries/0_stateless/01511_format_readable_timedelta.reference +++ b/tests/queries/0_stateless/01511_format_readable_timedelta.reference @@ -1,30 +1,30 @@ -1 second 1 second -2 seconds 2 seconds -7 seconds 7 seconds -20 seconds 20 seconds -54 seconds 54 seconds -2 minutes and 28 seconds 2 minutes and 28 seconds -6 minutes and 43 seconds 6 minutes and 43 seconds -18 minutes and 16 seconds 18 minutes and 16 seconds -49 minutes and 40 seconds 49 minutes and 40 seconds -2 hours, 15 minutes and 3 seconds 2 hours, 15 minutes and 3 seconds -6 hours, 7 minutes and 6 seconds 6 hours, 7 minutes and 6 seconds -16 hours, 37 minutes and 54 seconds 16 hours, 37 minutes and 54 seconds -1 day, 21 hours, 12 minutes and 34 seconds 1 day, 21 hours, 12 minutes and 34 seconds -5 days, 2 hours, 53 minutes and 33 seconds 5 days, 2 hours, 53 minutes and 33 seconds -13 days, 22 hours, 3 minutes and 24 seconds 13 days, 22 hours, 3 minutes and 24 seconds -1month, 6 days, 20 hours, 3 minutes and 37 seconds 1month, 6 days, 20 hours, 3 minutes and 37 seconds -3 months, 10 days, 20 hours, 21 minutes and 50 seconds 3 months, 10 days, 20 hours, 21 minutes and 50 seconds -9 months, 4 days, 13 hours, 42 minutes and 32 seconds 9 months, 4 days, 13 hours, 42 minutes and 32 seconds -2 years, 29 days, 22 hours, 52 minutes and 49 seconds 2 years, 29 days, 22 hours, 52 minutes and 49 seconds -5 years, 7 months, 26 days, 18 hours and 25 minutes 5 years, 7 months, 26 days, 18 hours and 25 minutes -15 years, 4 months, 18 days, 8 hours, 6 minutes and 35 seconds 15 years, 4 months, 18 days, 8 hours, 6 minutes and 35 seconds -41 years, 9 months, 24 days, 1 hour, 42 minutes and 14 seconds 41 years, 9 months, 24 days, 1 hour, 42 minutes and 14 seconds -113 years, 8 months, 3 days, 1 hour, 7 minutes and 26 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -309 years, 2 days, 1 hour, 50 minutes and 46 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -839 years, 11 months, 16 days, 1 hour, 28 minutes and 49 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -2283 years, 3 months, 3 days, 55 minutes and 37 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -6206 years, 6 months, 15 days, 23 hours, 57 minutes and 8 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -16871 years, 1month, 19 days, 17 hours, 56 minutes and 42 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -45860 years, 6 months, 3 days, 9 hours, 24 minutes and 52 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds -124661 years, 9 months, 14 days, 8 hours, 45 minutes and 42 seconds - 68 years, 1month, 4 days, 3 hours, 14 minutes and 8 seconds +1 5.5 seconds +60 5 minutes and 30 seconds +3600 5 hours and 30 minutes +86400 5 days and 12 hours +2592000 5 months, 12 days and 12 hours +31536000 5 years, 5 months and 30 days +minutes 1 5.5 seconds +minutes 60 5 minutes and 30 seconds +minutes 3600 330 minutes +minutes 86400 7920 minutes +minutes 2592000 237600 minutes +minutes 31536000 2890800 minutes +hours 1 5.5 seconds +hours 60 5 minutes and 30 seconds +hours 3600 5 hours and 30 minutes +hours 86400 132 hours +hours 2592000 3960 hours +hours 31536000 48180 hours +days 1 5.5 seconds +days 60 5 minutes and 30 seconds +days 3600 5 hours and 30 minutes +days 86400 5 days and 12 hours +days 2592000 165 days +days 31536000 2007 days and 12 hours +months 1 5.5 seconds +months 60 5 minutes and 30 seconds +months 3600 5 hours and 30 minutes +months 86400 5 days and 12 hours +months 2592000 5 months, 12 days and 12 hours +months 31536000 65 months and 25 days diff --git a/tests/queries/0_stateless/01511_format_readable_timedelta.sql b/tests/queries/0_stateless/01511_format_readable_timedelta.sql index a71742d540c..cf5781456ac 100755 --- a/tests/queries/0_stateless/01511_format_readable_timedelta.sql +++ b/tests/queries/0_stateless/01511_format_readable_timedelta.sql @@ -1,6 +1,19 @@ -WITH round(exp(number), 6) AS x, toUInt64(x) AS y, toInt32(x) AS z -SELECT - formatReadableTimeDelta(y), - formatReadableTimeDelta(z) -FROM system.numbers -LIMIT 30; +SELECT + arrayJoin([1, 60, 60*60, 60*60*24, 60*60*24*30, 60*60*24*365]) AS elapsed, + formatReadableTimeDelta(elapsed*5.5) AS time_delta; +SELECT + 'minutes' AS maximum_unit, + arrayJoin([1, 60, 60*60, 60*60*24, 60*60*24*30, 60*60*24*365]) AS elapsed, + formatReadableTimeDelta(elapsed*5.5, maximum_unit) AS time_delta; +SELECT + 'hours' AS maximum_unit, + arrayJoin([1, 60, 60*60, 60*60*24, 60*60*24*30, 60*60*24*365]) AS elapsed, + formatReadableTimeDelta(elapsed*5.5, maximum_unit) AS time_delta; +SELECT + 'days' AS maximum_unit, + arrayJoin([1, 60, 60*60, 60*60*24, 60*60*24*30, 60*60*24*365]) AS elapsed, + formatReadableTimeDelta(elapsed*5.5, maximum_unit) AS time_delta; +SELECT + 'months' AS maximum_unit, + arrayJoin([1, 60, 60*60, 60*60*24, 60*60*24*30, 60*60*24*365]) AS elapsed, + formatReadableTimeDelta(elapsed*5.5, maximum_unit) AS time_delta; From bab7c762bc681db045cf838a12913e2d4f3e8c1f Mon Sep 17 00:00:00 2001 From: filipe Date: Sat, 3 Oct 2020 20:38:21 -0300 Subject: [PATCH 090/441] fix style and pvs check --- src/Functions/FunctionsFormatting.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index ab5166bac46..809959d96ae 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -383,7 +383,7 @@ private: aux += hours * 3600; long long int minutes = maximum_unit_int < 2 ? 0 : (value - aux) / 60; aux += minutes * 60; - double seconds = maximum_unit_int < 1 ? 0 : value - aux; + double seconds = value - aux; std::vector parts; @@ -429,9 +429,9 @@ private: } String str_value; - for(size_t i=0; igetValue(); else throw Exception( - "Illegal column " + maximum_unit_const_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + "Illegal column " + arguments[1].get()->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) From 0436e0f8f145c331dcec224d478ac6d80ef60680 Mon Sep 17 00:00:00 2001 From: filipe Date: Sat, 3 Oct 2020 20:50:44 -0300 Subject: [PATCH 091/441] fix style and pvs check --- src/Functions/FunctionsFormatting.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index 809959d96ae..b2de00516cc 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -452,11 +452,12 @@ private: if (arguments.size() == 2) { const ColumnPtr & maximum_unit_column = block.getByPosition(arguments[1]).column; - if (const ColumnConst * maximum_unit_const_col = checkAndGetColumnConstStringOrFixedString(maximum_unit_column.get())) + const ColumnConst * maximum_unit_const_col = checkAndGetColumnConstStringOrFixedString(maximum_unit_column.get()); + if (maximum_unit_const_col) maximum_unit = maximum_unit_const_col->getValue(); else throw Exception( - "Illegal column " + arguments[1].get()->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + "Illegal column " + maximum_unit_const_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) From 00a354cd376a0fbb0fcf6aca1e6e1d7fbebadc34 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sun, 4 Oct 2020 22:00:56 +0400 Subject: [PATCH 092/441] Manually remove storages in reverse order in MultipleAccessStorage d-tor --- src/Access/MultipleAccessStorage.cpp | 9 +++++++++ src/Access/MultipleAccessStorage.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 8ddc7410d8d..792f4973a78 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,14 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_) { } +MultipleAccessStorage::~MultipleAccessStorage() +{ + auto storages = getStoragesPtr(); + for (auto storage : *storages | boost::adaptors::reversed) + { + removeStorage(storage); + } +} void MultipleAccessStorage::setStorages(const std::vector & storages) { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 36551f1cbc8..8844de8c029 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -18,6 +18,7 @@ public: using ConstStoragePtr = std::shared_ptr; MultipleAccessStorage(const String & storage_name_ = STORAGE_TYPE); + ~MultipleAccessStorage() override; const char * getStorageType() const override { return STORAGE_TYPE; } From 8263e62298ac8eb994feca546f6f6eaa75264074 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Sun, 4 Oct 2020 19:55:39 +0000 Subject: [PATCH 093/441] working copy (with some debug info) --- src/Storages/MergeTree/IMergeTreeDataPart.h | 3 +- .../MergeTree/ReplicatedMergeTreeBlockEntry.h | 6 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 14 ++-- .../ReplicatedMergeTreeCleanupThread.cpp | 18 +++- .../ReplicatedMergeTreePartHeader.cpp | 8 +- .../MergeTree/ReplicatedMergeTreePartHeader.h | 1 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 31 +++++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 2 +- .../ReplicatedMergeTreeQuorumStatusEntry.h | 2 +- .../ReplicatedMergeTreeRestartingThread.cpp | 30 +++++++ src/Storages/StorageReplicatedMergeTree.cpp | 83 ++++++++++++++----- src/Storages/StorageReplicatedMergeTree.h | 3 +- 12 files changed, 156 insertions(+), 45 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 78daf6c9017..b25e6d570f5 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -60,7 +60,6 @@ public: using Type = MergeTreeDataPartType; - IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, @@ -349,6 +348,8 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; + std::optional block_id; + protected: /// Total size of all columns, calculated once in calcuateColumnSizesOnDisk diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h index 2677323ae89..97496538b16 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h @@ -31,10 +31,12 @@ struct ReplicatedMergeTreeBlockEntry void writeText(WriteBuffer & out) const { - out << part_name << "\n"; + out << part_name; - if (quorum_status) + if (quorum_status) { + out << "\n"; quorum_status->writeText(out); + } } void readText(ReadBuffer & in) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index f56176f4ff6..d502ecb9202 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -101,7 +101,7 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe */ auto quorum_status = quorum_status_future.get(); - if (quorum_status.error != Coordination::Error::ZNONODE) + if (quorum_status.error != Coordination::Error::ZNONODE && !quorum_parallel) throw Exception("Quorum for previous write has not been satisfied yet. Status: " + quorum_status.data, ErrorCodes::UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE); @@ -172,6 +172,7 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) try { + LOG_ERROR(log, "need to send here block_id somehow"); commitPart(zookeeper, part, block_id); /// Set a special error code if the block is duplicate @@ -342,8 +343,9 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( /// Note: race condition with DROP PARTITION operation is possible. User will get "No node" exception and it is Ok. existing_part_name = zookeeper->get(storage.zookeeper_path + "/blocks/" + block_id); + block_entry.fromString(existing_part_name); /// If it exists on our replica, ignore it. - if (storage.getActiveContainingPart(existing_part_name)) + if (storage.getActiveContainingPart(block_entry.part_name)) { LOG_INFO(log, "Block with ID {} already exists locally as part {}; ignoring it.", block_id, existing_part_name); part->is_duplicate = true; @@ -370,6 +372,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( } /// Information about the part. + LOG_INFO(log, "getCommitPartOps from ...OutputStream"); storage.getCommitPartOps(ops, part, block_id_path, block_entry); MergeTreeData::Transaction transaction(storage); /// If you can not add a part to ZK, we'll remove it back from the working set. @@ -484,9 +487,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( if (is_already_existing_part) { /// We get duplicate part without fetch - /// ALEXELEXA - /// should reset here something, after thinking in TODO - storage.updateQuorum(part->name, part->info.block_id); + storage.updateQuorum(part->name); } /// We are waiting for quorum to be satisfied. @@ -511,7 +512,8 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( if (quorum_parallel) { block_entry.fromString(value); - if (block_entry.part_name != part->name) + // quorum_status empty if quorum reached + if (block_entry.part_name != part->name || !block_entry.quorum_status) break; } else diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 4df47915048..4f2cc0adb85 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -342,13 +342,25 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold); + zkutil::AsyncResponses try_get_futures; zkutil::AsyncResponses try_remove_futures; for (auto it = first_outdated_block; it != timed_blocks.end(); ++it) { String path = storage.zookeeper_path + "/blocks/" + it->node; - /// ALEXELEXA - /// should check somehow is it quorum block - try_remove_futures.emplace_back(path, zookeeper->asyncTryRemove(path)); + try_get_futures.emplace_back(path, zookeeper->asyncTryGet(path)); + } + + /// Don't delete blocks that are still reaching quorum + for (auto & pair : try_get_futures) + { + const String & path = pair.first; + auto response = pair.second.get(); + if (response.error == Coordination::Error::ZOK) + { + ReplicatedMergeTreeBlockEntry block(response.data); + if (!block.quorum_status) + try_remove_futures.emplace_back(path, zookeeper->asyncTryRemove(path)); + } } for (auto & pair : try_remove_futures) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp index db52f05c0c9..932c4497d30 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp @@ -51,7 +51,11 @@ void ReplicatedMergeTreePartHeader::read(ReadBuffer & in) checksums.deserializeWithoutHeader(in); if (!in.eof()) - in >> "block_id: " >> block_id.value() >> "\n"; + { + String block_id_; + in >> "\nblock_id: " >> block_id_; + block_id = block_id_; + } } ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromString(const String & str) @@ -68,7 +72,7 @@ void ReplicatedMergeTreePartHeader::write(WriteBuffer & out) const out.write(columns_hash.data(), columns_hash.size()); checksums.serializeWithoutHeader(out); if (block_id) - out << "block_id " << block_id.value() << "\n"; + out << "\nblock_id: " << block_id.value(); } String ReplicatedMergeTreePartHeader::toString() const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h index d0b8c11dfb6..c67aba29f32 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h @@ -40,6 +40,7 @@ public: const std::array & getColumnsHash() const { return columns_hash; } const MinimalisticDataPartChecksums & getChecksums() const { return checksums; } + const std::optional & getBlockID() const { return block_id; } private: ReplicatedMergeTreePartHeader(std::array columns_hash_, MinimalisticDataPartChecksums checksums_, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index ef7ebead966..128fcb66f96 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -1697,9 +1698,6 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( prev_virtual_parts = queue.virtual_parts; } - /// Load current quorum status. - auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); - /// Load current inserts std::unordered_set lock_holder_paths; for (const String & entry : zookeeper->getChildren(queue.zookeeper_path + "/temp")) @@ -1751,15 +1749,32 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( merges_version = queue_.pullLogsToQueue(zookeeper); + /// Load current quorum status. + auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); Coordination::GetResponse quorum_status_response = quorum_status_future.get(); if (quorum_status_response.error == Coordination::Error::ZOK) { ReplicatedMergeTreeQuorumEntry quorum_status; quorum_status.fromString(quorum_status_response.data); - inprogress_quorum_part = quorum_status.part_name; + inprogress_quorum_parts.insert(quorum_status.part_name); } else - inprogress_quorum_part.clear(); + inprogress_quorum_parts.clear(); + + Strings partitions = zookeeper->getChildren(queue.replica_path + "/parts"); + for (const String & partition : partitions) + { + auto header = ReplicatedMergeTreePartHeader::fromString(zookeeper->get(queue.replica_path + "/parts/" + partition)); + if (header.getBlockID()) + { + ReplicatedMergeTreeBlockEntry block(zookeeper->get(queue.zookeeper_path + "/blocks/" + *header.getBlockID())); + if (partition != block.part_name) + throw Exception("partition " + partition + " contain block_id " + *header.getBlockID() + + " and block_id contain another partition " + block.part_name, ErrorCodes::LOGICAL_ERROR); + if (block.quorum_status) + inprogress_quorum_parts.insert(block.part_name); + } + } } bool ReplicatedMergeTreeMergePredicate::operator()( @@ -1821,7 +1836,7 @@ bool ReplicatedMergeTreeMergePredicate::canMergeTwoParts( for (const MergeTreeData::DataPartPtr & part : {left, right}) { - if (part->name == inprogress_quorum_part) + if (inprogress_quorum_parts.count(part->name)) { if (out_reason) *out_reason = "Quorum insert for part " + part->name + " is currently in progress"; @@ -1916,7 +1931,7 @@ bool ReplicatedMergeTreeMergePredicate::canMergeSinglePart( const MergeTreeData::DataPartPtr & part, String * out_reason) const { - if (part->name == inprogress_quorum_part) + if (inprogress_quorum_parts.count(part->name)) { if (out_reason) *out_reason = "Quorum insert for part " + part->name + " is currently in progress"; @@ -1957,7 +1972,7 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir /// We cannot mutate part if it's being inserted with quorum and it's not /// already reached. - if (part->name == inprogress_quorum_part) + if (inprogress_quorum_parts.count(part->name)) return {}; std::lock_guard lock(queue.state_mutex); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 88a61f50225..fb2733db667 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -475,7 +475,7 @@ private: std::unordered_map> committing_blocks; /// Quorum state taken at some later time than prev_virtual_parts. - String inprogress_quorum_part; + std::unordered_set inprogress_quorum_parts; int32_t merges_version = -1; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h index bb3cb83b9a1..95987ea13b1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h @@ -69,7 +69,7 @@ struct ReplicatedMergeTreeQuorumStatusEntry bool isQuorumReached() { - return required_number_of_replicas <= replicas.size(); + return required_number_of_replicas && required_number_of_replicas <= replicas.size(); } }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 5a89906dd3f..908da215d42 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include @@ -236,6 +238,34 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() storage.updateQuorum(quorum_entry.part_name); } } + + Strings partitions; + if (zookeeper->tryGetChildren(storage.replica_path + "/parts/", partitions) == Coordination::Error::ZOK) + { + for (auto & partition : partitions) + { + String part_str, block_str; + if (!zookeeper->tryGet(storage.replica_path + "/parts/" + partition, part_str)) + continue; + + auto header = ReplicatedMergeTreePartHeader::fromString(part_str); + if (!header.getBlockID() || !zookeeper->tryGet(storage.zookeeper_path + "/blocks/" + *header.getBlockID(), block_str)) + continue; + + ReplicatedMergeTreeBlockEntry block_entry(block_str); + if (block_entry.part_name != partition) + { + LOG_WARNING(log, "partition {} contain block_id {} and block_id contain another partition {}", partition, *header.getBlockID(), block_entry.part_name); + continue; + } + + if (block_entry.quorum_status && !block_entry.quorum_status->replicas.count(storage.replica_name)) + { + LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", block_entry.part_name); + storage.updateQuorum(block_entry.part_name, header.getBlockID()); + } + } + } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1df419b6dac..d09cb3f359b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1180,6 +1181,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { + LOG_DEBUG(log, "creating empty! (2)"); ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( @@ -1700,30 +1702,37 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (replica.empty()) { - Coordination::Stat quorum_stat; + Coordination::Stat quorum_stat, block_stat; String quorum_path = zookeeper_path + "/quorum/status"; - String quorum_str = zookeeper->get(quorum_path, &quorum_stat); + String block_path = entry.block_id.empty() ? "" : zookeeper_path + "/blocks/" + entry.block_id; + String quorum_str, parallel_quorum_str; ReplicatedMergeTreeQuorumEntry quorum_entry; - quorum_entry.fromString(quorum_str); + ReplicatedMergeTreeBlockEntry block_entry; - if (quorum_entry.part_name == entry.new_part_name) + if (zookeeper->tryGet(quorum_path, quorum_str, &quorum_stat)) + quorum_entry.fromString(quorum_str); + if (!block_path.empty() && zookeeper->tryGet(block_path, parallel_quorum_str, &block_stat)) + block_entry.fromString(parallel_quorum_str); + bool is_parallel_quorum = (block_entry.part_name == entry.new_part_name); + + if (quorum_entry.part_name == entry.new_part_name || is_parallel_quorum) { - ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); - auto part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); - if (part_info.min_block != part_info.max_block) throw Exception("Logical error: log entry with quorum for part covering more than one block number", ErrorCodes::LOGICAL_ERROR); + if (!is_parallel_quorum) + ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); + ops.emplace_back(zkutil::makeCreateRequest( zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name, "", zkutil::CreateMode::Persistent)); /// Deleting from `blocks`. - if (!entry.block_id.empty() && zookeeper->exists(zookeeper_path + "/blocks/" + entry.block_id)) - ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/blocks/" + entry.block_id, -1)); + if (!block_path.empty() && zookeeper->exists(block_path)) + ops.emplace_back(zkutil::makeRemoveRequest(block_path, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -1757,8 +1766,12 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) try { + std::optional block_id; + if (!entry.block_id.empty()) + block_id = entry.block_id; String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; - if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum)) + LOG_ERROR(log, "if (!fetchPart(part_name, metadata_snapshot... "); + if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum, nullptr, block_id)) return false; } catch (Exception & e) @@ -2111,6 +2124,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (interserver_scheme != address.scheme) throw Exception("Interserver schemas are different '" + interserver_scheme + "' != '" + address.scheme + "', can't fetch part from " + address.host, ErrorCodes::LOGICAL_ERROR); + LOG_ERROR(log, "WOW: part_desc->res_part = fetcher.fetchPart("); part_desc->res_part = fetcher.fetchPart( metadata_snapshot, part_desc->found_new_part_name, source_replica_path, address.host, address.replication_port, timeouts, user, password, interserver_scheme, false, TMP_PREFIX + "fetch_"); @@ -2142,6 +2156,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) for (PartDescriptionPtr & part_desc : final_parts) { renameTempPartAndReplace(part_desc->res_part, nullptr, &transaction); + LOG_INFO(log, "getCommitPartOps from Storage...::executeReplaceRange"); getCommitPartOps(ops, part_desc->res_part); if (ops.size() > zkutil::MULTI_BATCH_SIZE) @@ -3039,6 +3054,8 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; const String quorum_parallel_status_path = block_id ? zookeeper_path + "/blocks/" + *block_id : ""; + LOG_ERROR(log, "updating quorum! {}", quorum_parallel_status_path); + /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. String value; Coordination::Stat stat; @@ -3048,13 +3065,18 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st ReplicatedMergeTreeBlockEntry block_entry; ReplicatedMergeTreeQuorumEntry quorum_entry; + LOG_ERROR(log, "trying to get something"); if (zookeeper->tryGet(quorum_status_path, value, &stat)) { + LOG_ERROR(log, "got /quorum/status"); + LOG_ERROR(log, "got from /quorum/status: {}", value); quorum_entry.fromString(value); quorum_entry.status.replicas.insert(replica_name); } else if (block_id && zookeeper->tryGet(quorum_parallel_status_path, value, &stat)) { + LOG_ERROR(log, "got from {}", quorum_parallel_status_path); + LOG_ERROR(log, "got from block: {}", value); block_entry.fromString(value); // The quorum has already been achieved if (!block_entry.quorum_status) @@ -3062,20 +3084,24 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st is_parallel = true; block_entry.quorum_status->replicas.insert(replica_name); + LOG_ERROR(log, "block data became: {}", block_entry.toString()); if (block_entry.quorum_status->isQuorumReached()) block_entry.quorum_status.reset(); + LOG_ERROR(log, "AFTER checkReached: block data became: {}", block_entry.toString()); } else break; if (quorum_entry.part_name != part_name && block_entry.part_name != part_name) { + LOG_ERROR(log, "another part_names {} {} {}", quorum_entry.part_name, block_entry.part_name, part_name); /// The quorum has already been achieved. Moreover, another INSERT with a quorum has already started. break; } if (quorum_entry.status.isQuorumReached()) { + LOG_ERROR(log, "normal quorum reached"); /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. Coordination::Requests ops; @@ -3119,12 +3145,15 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st else { Coordination::Error code; - if (is_parallel) + LOG_ERROR(log, "else"); + if (!is_parallel) { + LOG_ERROR(log, "!is_parallel"); /// We update the node, registering there one more replica. code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); } else { + LOG_ERROR(log, "updating {} : {}", quorum_parallel_status_path, block_entry.toString()); /// Update parallel quorum. It also might be reached here. code = zookeeper->trySet(quorum_parallel_status_path, block_entry.toString(), stat.version); } @@ -3203,7 +3232,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, - const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_) + const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_, const std::optional & block_id) { auto zookeeper = zookeeper_ ? zookeeper_ : getZooKeeper(); const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -3306,6 +3335,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora + "' != '" + address.scheme + "', can't fetch part from " + address.host, ErrorCodes::LOGICAL_ERROR); + LOG_ERROR(log, "WOW: return fetcher.fetchPart("); return fetcher.fetchPart( metadata_snapshot, part_name, source_replica_path, address.host, address.replication_port, @@ -3333,7 +3363,10 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. */ if (quorum) - updateQuorum(part_name); + { + LOG_ERROR(log, "updateQuorum from Storage...::3349"); + updateQuorum(part_name, block_id); + } merge_selecting_task->schedule(); @@ -4830,6 +4863,7 @@ void StorageReplicatedMergeTree::fetchPartition( try { + LOG_ERROR(log, "OH:fetched = fetchPart(part, ..."); fetched = fetchPart(part, metadata_snapshot, best_replica_path, true, 0, zookeeper); } catch (const DB::Exception & e) @@ -5272,11 +5306,11 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( continue; /// ALEXELEXA - /// should parse it here using another way - ReadBufferFromString buf(result.data); + ReplicatedMergeTreeBlockEntry block(result.data); + ReadBufferFromString buf(block.part_name); Int64 block_num = 0; bool parsed = tryReadIntText(block_num, buf) && buf.eof(); - if (!parsed || (min_block_num <= block_num && block_num <= max_block_num)) + if ((!parsed || (min_block_num <= block_num && block_num <= max_block_num)) && !block.quorum_status) to_delete_futures.emplace_back(path, zookeeper.asyncTryRemove(path)); } @@ -5422,6 +5456,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( Coordination::Requests ops; for (size_t i = 0; i < dst_parts.size(); ++i) { + LOG_INFO(log, "getCommitPartOps from Storage...::replacePartitionFrom"); getCommitPartOps(ops, dst_parts[i], block_id_paths[i]); ephemeral_locks[i].getUnlockOps(ops); @@ -5612,6 +5647,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Coordination::Requests ops; for (size_t i = 0; i < dst_parts.size(); ++i) { + LOG_INFO(log, "getCommitPartOps from Storage...::movePartitionToTable"); dest_table_storage->getCommitPartOps(ops, dst_parts[i], block_id_paths[i]); ephemeral_locks[i].getUnlockOps(ops); @@ -5709,20 +5745,27 @@ void StorageReplicatedMergeTree::getCommitPartOps( } /// Information about the part, in the replica + String part_str, block_id = !block_id_path.empty() && block_entry.quorum_status ? block_id_path.substr(block_id_path.find_last_of('/') + 1) : ""; if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { + if (!block_id.empty()) + part_str = ReplicatedMergeTreePartHeader::fromColumnsChecksumsBlockID(part->getColumns(), part->checksums, block_id).toString(); + else + part_str = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(); + ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, - ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), + part_str, zkutil::CreateMode::Persistent)); - LOG_DEBUG(log, "ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(): {}", - ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString()); + LOG_DEBUG(log, "ReplicatedMergeTreePartHeader::..: {}", part_str); } else { + /// ALEXELEXA think! + LOG_DEBUG(log, "creating empty!!!"); ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, - "", + "", /// how it reads? zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name + "/columns", diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index daf8b712a6c..8014ed600ea 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -486,7 +486,8 @@ private: const String & replica_path, bool to_detached, size_t quorum, - zkutil::ZooKeeper::Ptr zookeeper_ = nullptr); + zkutil::ZooKeeper::Ptr zookeeper_ = nullptr, + const std::optional & block_id = std::nullopt); /// Required only to avoid races between executeLogEntry and fetchPartition std::unordered_set currently_fetching_parts; From 7f477197681d152ac64a7cff55726e672d8010d2 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sun, 4 Oct 2020 23:55:58 +0400 Subject: [PATCH 094/441] Refactor exception handling in login() et al. Simplify LDAPClient and LDAPAccessStorage --- src/Access/IAccessStorage.cpp | 31 ++++++++++-- src/Access/IAccessStorage.h | 2 + src/Access/LDAPAccessStorage.cpp | 70 ++++++++++++++++------------ src/Access/LDAPAccessStorage.h | 2 + src/Access/LDAPClient.cpp | 59 ++++------------------- src/Access/LDAPClient.h | 1 - src/Access/LDAPParams.h | 1 + src/Access/MultipleAccessStorage.cpp | 2 +- 8 files changed, 82 insertions(+), 86 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index e5170221e18..cb490d488c8 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -14,6 +14,8 @@ namespace ErrorCodes extern const int ACCESS_ENTITY_ALREADY_EXISTS; extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; + extern const int WRONG_PASSWORD; + extern const int IP_ADDRESS_NOT_ALLOWED; extern const int AUTHENTICATION_FAILED; extern const int LOGICAL_ERROR; } @@ -420,7 +422,14 @@ UUID IAccessStorage::login( const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { - return loginImpl(user_name, password, address, external_authenticators); + try { + return loginImpl(user_name, password, address, external_authenticators); + } + catch (...) + { + tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); + throwCannotAuthenticate(user_name); + } } @@ -434,11 +443,16 @@ UUID IAccessStorage::loginImpl( { if (auto user = tryRead(*id)) { - if (isPasswordCorrectImpl(*user, password, external_authenticators) && isAddressAllowedImpl(*user, address)) - return *id; + if (!isPasswordCorrectImpl(*user, password, external_authenticators)) + throwInvalidPassword(); + + if (!isAddressAllowedImpl(*user, address)) + throwAddressNotAllowed(address); + + return *id; } } - throwCannotAuthenticate(user_name); + throwNotFound(EntityType::USER, user_name); } @@ -554,6 +568,15 @@ void IAccessStorage::throwReadonlyCannotRemove(EntityType type, const String & n ErrorCodes::ACCESS_STORAGE_READONLY); } +void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address) +{ + throw Exception("Connections from " + address.toString() + " are not allowed", ErrorCodes::IP_ADDRESS_NOT_ALLOWED); +} + +void IAccessStorage::throwInvalidPassword() +{ + throw Exception("Invalid password", ErrorCodes::WRONG_PASSWORD); +} void IAccessStorage::throwCannotAuthenticate(const String & user_name) { diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 5a86e817fb2..059c6103f6a 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -182,6 +182,8 @@ protected: [[noreturn]] void throwReadonlyCannotInsert(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const; + [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address); + [[noreturn]] static void throwInvalidPassword(); [[noreturn]] static void throwCannotAuthenticate(const String & user_name); using Notification = std::tuple; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 0142b98f1bb..9b1c6a48a13 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -230,42 +230,50 @@ bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { std::scoped_lock lock(mutex); - try + auto id = memory_storage.find(user_name); + if (id) { - auto id = memory_storage.find(user_name); - if (id) - { - auto user = memory_storage.tryRead(*id); - if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) - return *id; - } - else - { - // User does not exist, so we create one, and will add it if authentication is successful. - auto user = std::make_shared(); - user->setName(user_name); - user->authentication = Authentication(Authentication::Type::LDAP_SERVER); - user->authentication.setServerName(ldap_server); + auto user = memory_storage.read(*id); - if (isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) - { - for (const auto& role_name : default_role_names) - { - std::optional role_id = access_control_manager->find(role_name); - if (!role_id) - throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } - return memory_storage.insert(user); - } - } + if (!isPasswordCorrectImpl(*user, password, external_authenticators)) + throwInvalidPassword(); + + if (!isAddressAllowedImpl(*user, address)) + throwAddressNotAllowed(address); + + return *id; } - catch (...) + else { - tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + getStorageName() + "'"); + // User does not exist, so we create one, and will add it if authentication is successful. + auto user = std::make_shared(); + user->setName(user_name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + if (!isPasswordCorrectImpl(*user, password, external_authenticators)) + throwInvalidPassword(); + + if (!isAddressAllowedImpl(*user, address)) + throwAddressNotAllowed(address); + + for (const auto& role_name : default_role_names) + { + std::optional role_id = access_control_manager->find(role_name); + if (!role_id) + throwDefaultRoleNotFound(role_name); + + roles_of_interest.insert(role_id.value()); + user->granted_roles.grant(role_id.value()); + } + + return memory_storage.insert(user); } - throwCannotAuthenticate(user_name); +} + +void LDAPAccessStorage::throwDefaultRoleNotFound(const String & role_name) +{ + throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); } } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 35444944ec6..02c44a8d400 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -55,6 +55,8 @@ private: void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + [[noreturn]] static void throwDefaultRoleNotFound(const String & role_name); + mutable std::recursive_mutex mutex; AccessControlManager * access_control_manager = nullptr; String ldap_server; diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index d3231f62f3b..a3223902361 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -106,14 +106,6 @@ void LDAPClient::openConnection() { std::scoped_lock lock(ldap_global_mutex); - const bool graceful_bind_failure = false; - diag(openConnection(graceful_bind_failure)); -} - -int LDAPClient::openConnection(const bool graceful_bind_failure) -{ - std::scoped_lock lock(ldap_global_mutex); - closeConnection(); { @@ -244,8 +236,6 @@ int LDAPClient::openConnection(const bool graceful_bind_failure) if (params.enable_tls == LDAPServerParams::TLSEnable::YES_STARTTLS) diag(ldap_start_tls_s(handle, nullptr, nullptr)); - int rc = LDAP_OTHER; - switch (params.sasl_mechanism) { case LDAPServerParams::SASLMechanism::SIMPLE: @@ -256,16 +246,15 @@ int LDAPClient::openConnection(const bool graceful_bind_failure) cred.bv_val = const_cast(params.password.c_str()); cred.bv_len = params.password.size(); - rc = ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr); - - if (!graceful_bind_failure) - diag(rc); + diag(ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr)); break; } + default: + { + throw Exception("Unknown SASL mechanism", ErrorCodes::LDAP_ERROR); + } } - - return rc; } void LDAPClient::closeConnection() noexcept @@ -286,39 +275,16 @@ bool LDAPSimpleAuthClient::check() if (params.user.empty()) throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); + // Silently reject authentication attempt if the password is empty as if it didn't match. if (params.password.empty()) - return false; // Silently reject authentication attempt if the password is empty as if it didn't match. + return false; SCOPE_EXIT({ closeConnection(); }); - const bool graceful_bind_failure = true; - const auto rc = openConnection(graceful_bind_failure); + // Will throw on any error, including invalid credentials. + openConnection(); - bool result = false; - - switch (rc) - { - case LDAP_SUCCESS: - { - result = true; - break; - } - - case LDAP_INVALID_CREDENTIALS: - { - result = false; - break; - } - - default: - { - result = false; - diag(rc); - break; - } - } - - return result; + return true; } #else // USE_LDAP @@ -333,11 +299,6 @@ void LDAPClient::openConnection() throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } -int LDAPClient::openConnection(const bool) -{ - throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); -} - void LDAPClient::closeConnection() noexcept { } diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index b117ed9a026..777c87c5b94 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -32,7 +32,6 @@ public: protected: MAYBE_NORETURN void diag(const int rc); MAYBE_NORETURN void openConnection(); - int openConnection(const bool graceful_bind_failure = false); void closeConnection() noexcept; protected: diff --git a/src/Access/LDAPParams.h b/src/Access/LDAPParams.h index 2168ce45203..eeadba6bc01 100644 --- a/src/Access/LDAPParams.h +++ b/src/Access/LDAPParams.h @@ -42,6 +42,7 @@ struct LDAPServerParams enum class SASLMechanism { + UNKNOWN, SIMPLE }; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 792f4973a78..eb9b142e112 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -425,7 +425,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p throw; } } - throwCannotAuthenticate(user_name); + throwNotFound(EntityType::USER, user_name); } From 1eb8ecf050d071506304c92fcd295167abc8a0bc Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sun, 4 Oct 2020 23:56:25 +0400 Subject: [PATCH 095/441] Fix compilation --- src/Access/MultipleAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index eb9b142e112..516042b5af5 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -30,8 +30,8 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_) MultipleAccessStorage::~MultipleAccessStorage() { - auto storages = getStoragesPtr(); - for (auto storage : *storages | boost::adaptors::reversed) + const auto storages = getStoragesPtr(); + for (const auto & storage : *storages | boost::adaptors::reversed) { removeStorage(storage); } From ff8cbe6745403c40a9c6c5df511765b208746f80 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Sun, 4 Oct 2020 20:19:05 +0000 Subject: [PATCH 096/441] remove debug things --- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 -- src/Storages/MergeTree/MergeTreePartInfo.h | 3 -- .../ReplicatedMergeTreeBlockOutputStream.cpp | 7 ---- .../ReplicatedMergeTreeCleanupThread.cpp | 2 -- src/Storages/StorageReplicatedMergeTree.cpp | 36 ++++--------------- 5 files changed, 6 insertions(+), 44 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index b25e6d570f5..e36d1ee5b08 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -348,8 +348,6 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; - std::optional block_id; - protected: /// Total size of all columns, calculated once in calcuateColumnSizesOnDisk diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index d538418c29b..416308861b7 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -19,8 +18,6 @@ struct MergeTreePartInfo Int64 max_block = 0; UInt32 level = 0; Int64 mutation = 0; /// If the part has been mutated or contains mutated parts, is equal to mutation version number. - std::optional block_id; /// hex if write with quorum and min_block == max_block -/// shouldn't be here... MergeTreePartInfo() = default; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index d502ecb9202..67ce355bb7f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -172,7 +172,6 @@ void ReplicatedMergeTreeBlockOutputStream::write(const Block & block) try { - LOG_ERROR(log, "need to send here block_id somehow"); commitPart(zookeeper, part, block_id); /// Set a special error code if the block is duplicate @@ -258,10 +257,6 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( part->info.min_block = block_number; part->info.max_block = block_number; - - /// ALEXELEXA - /// somehow need to send this block_if to part node. TODO - part->info.block_id = block_id; part->info.level = 0; part->name = part->getNewName(part->info); @@ -314,7 +309,6 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( quorum_info.status_path, quorum_entry.toString(), zkutil::CreateMode::Persistent)); - } else block_entry.quorum_status = status_entry; @@ -372,7 +366,6 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( } /// Information about the part. - LOG_INFO(log, "getCommitPartOps from ...OutputStream"); storage.getCommitPartOps(ops, part, block_id_path, block_entry); MergeTreeData::Transaction transaction(storage); /// If you can not add a part to ZK, we'll remove it back from the working set. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 4f2cc0adb85..03d920b2033 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -70,8 +70,6 @@ void ReplicatedMergeTreeCleanupThread::iterate() if (storage.is_leader) { clearOldLogs(); - /// ALEXELEXA - /// may be just remove it? clearOldBlocks(); clearOldMutations(); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d09cb3f359b..aece4b26130 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1177,11 +1177,12 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: { ops.emplace_back(zkutil::makeCreateRequest( part_path, local_part_header.toString(), zkutil::CreateMode::Persistent)); - LOG_DEBUG(log, "local_part_header.toString(): {}", local_part_header.toString()); } else { - LOG_DEBUG(log, "creating empty! (2)"); + /// ALEXELEX + /// is this a valid format or an old one? + /// should we support it for quorum parallel inserts? ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( @@ -1770,7 +1771,6 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (!entry.block_id.empty()) block_id = entry.block_id; String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; - LOG_ERROR(log, "if (!fetchPart(part_name, metadata_snapshot... "); if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum, nullptr, block_id)) return false; } @@ -2124,7 +2124,6 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) if (interserver_scheme != address.scheme) throw Exception("Interserver schemas are different '" + interserver_scheme + "' != '" + address.scheme + "', can't fetch part from " + address.host, ErrorCodes::LOGICAL_ERROR); - LOG_ERROR(log, "WOW: part_desc->res_part = fetcher.fetchPart("); part_desc->res_part = fetcher.fetchPart( metadata_snapshot, part_desc->found_new_part_name, source_replica_path, address.host, address.replication_port, timeouts, user, password, interserver_scheme, false, TMP_PREFIX + "fetch_"); @@ -2156,7 +2155,6 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) for (PartDescriptionPtr & part_desc : final_parts) { renameTempPartAndReplace(part_desc->res_part, nullptr, &transaction); - LOG_INFO(log, "getCommitPartOps from Storage...::executeReplaceRange"); getCommitPartOps(ops, part_desc->res_part); if (ops.size() > zkutil::MULTI_BATCH_SIZE) @@ -3054,8 +3052,6 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; const String quorum_parallel_status_path = block_id ? zookeeper_path + "/blocks/" + *block_id : ""; - LOG_ERROR(log, "updating quorum! {}", quorum_parallel_status_path); - /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. String value; Coordination::Stat stat; @@ -3065,18 +3061,13 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st ReplicatedMergeTreeBlockEntry block_entry; ReplicatedMergeTreeQuorumEntry quorum_entry; - LOG_ERROR(log, "trying to get something"); if (zookeeper->tryGet(quorum_status_path, value, &stat)) { - LOG_ERROR(log, "got /quorum/status"); - LOG_ERROR(log, "got from /quorum/status: {}", value); quorum_entry.fromString(value); quorum_entry.status.replicas.insert(replica_name); } else if (block_id && zookeeper->tryGet(quorum_parallel_status_path, value, &stat)) { - LOG_ERROR(log, "got from {}", quorum_parallel_status_path); - LOG_ERROR(log, "got from block: {}", value); block_entry.fromString(value); // The quorum has already been achieved if (!block_entry.quorum_status) @@ -3084,24 +3075,20 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st is_parallel = true; block_entry.quorum_status->replicas.insert(replica_name); - LOG_ERROR(log, "block data became: {}", block_entry.toString()); if (block_entry.quorum_status->isQuorumReached()) block_entry.quorum_status.reset(); - LOG_ERROR(log, "AFTER checkReached: block data became: {}", block_entry.toString()); } else break; if (quorum_entry.part_name != part_name && block_entry.part_name != part_name) { - LOG_ERROR(log, "another part_names {} {} {}", quorum_entry.part_name, block_entry.part_name, part_name); /// The quorum has already been achieved. Moreover, another INSERT with a quorum has already started. break; } if (quorum_entry.status.isQuorumReached()) { - LOG_ERROR(log, "normal quorum reached"); /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. Coordination::Requests ops; @@ -3145,15 +3132,12 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st else { Coordination::Error code; - LOG_ERROR(log, "else"); if (!is_parallel) { - LOG_ERROR(log, "!is_parallel"); /// We update the node, registering there one more replica. code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); } else { - LOG_ERROR(log, "updating {} : {}", quorum_parallel_status_path, block_entry.toString()); /// Update parallel quorum. It also might be reached here. code = zookeeper->trySet(quorum_parallel_status_path, block_entry.toString(), stat.version); } @@ -3335,7 +3319,6 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora + "' != '" + address.scheme + "', can't fetch part from " + address.host, ErrorCodes::LOGICAL_ERROR); - LOG_ERROR(log, "WOW: return fetcher.fetchPart("); return fetcher.fetchPart( metadata_snapshot, part_name, source_replica_path, address.host, address.replication_port, @@ -3363,10 +3346,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. */ if (quorum) - { - LOG_ERROR(log, "updateQuorum from Storage...::3349"); updateQuorum(part_name, block_id); - } merge_selecting_task->schedule(); @@ -4863,7 +4843,6 @@ void StorageReplicatedMergeTree::fetchPartition( try { - LOG_ERROR(log, "OH:fetched = fetchPart(part, ..."); fetched = fetchPart(part, metadata_snapshot, best_replica_path, true, 0, zookeeper); } catch (const DB::Exception & e) @@ -5305,7 +5284,6 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( if (result.error == Coordination::Error::ZNONODE) continue; - /// ALEXELEXA ReplicatedMergeTreeBlockEntry block(result.data); ReadBufferFromString buf(block.part_name); Int64 block_num = 0; @@ -5456,7 +5434,6 @@ void StorageReplicatedMergeTree::replacePartitionFrom( Coordination::Requests ops; for (size_t i = 0; i < dst_parts.size(); ++i) { - LOG_INFO(log, "getCommitPartOps from Storage...::replacePartitionFrom"); getCommitPartOps(ops, dst_parts[i], block_id_paths[i]); ephemeral_locks[i].getUnlockOps(ops); @@ -5647,7 +5624,6 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta Coordination::Requests ops; for (size_t i = 0; i < dst_parts.size(); ++i) { - LOG_INFO(log, "getCommitPartOps from Storage...::movePartitionToTable"); dest_table_storage->getCommitPartOps(ops, dst_parts[i], block_id_paths[i]); ephemeral_locks[i].getUnlockOps(ops); @@ -5761,11 +5737,11 @@ void StorageReplicatedMergeTree::getCommitPartOps( } else { - /// ALEXELEXA think! - LOG_DEBUG(log, "creating empty!!!"); + /// ALEXELEXA + /// also. should we support it? ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, - "", /// how it reads? + "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name + "/columns", From 2fc6a4ea9c0bd1af1ce08c88ca3113928c705929 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 5 Oct 2020 00:24:09 +0400 Subject: [PATCH 097/441] Add log_and_mask_exceptions flag to login() --- src/Access/IAccessStorage.cpp | 6 +++++- src/Access/IAccessStorage.h | 2 +- src/Access/MultipleAccessStorage.cpp | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index cb490d488c8..b21527e48f1 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -420,13 +420,17 @@ UUID IAccessStorage::login( const String & user_name, const String & password, const Poco::Net::IPAddress & address, - const ExternalAuthenticators & external_authenticators) const + const ExternalAuthenticators & external_authenticators, + bool log_and_mask_exceptions) const { try { return loginImpl(user_name, password, address, external_authenticators); } catch (...) { + if (!log_and_mask_exceptions) + throw; + tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); throwCannotAuthenticate(user_name); } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 059c6103f6a..93c97144cda 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -144,7 +144,7 @@ public: /// Finds an user, check its password and returns the ID of the user. /// Throws an exception if no such user or password is incorrect. - UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const; + UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool log_and_mask_exceptions = true) const; /// Returns the ID of an user who has logged in (maybe on another node). /// The function assumes that the password has been already checked somehow, so we can skip checking it now. diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 516042b5af5..32aa8c50159 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -409,7 +409,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p { try { - auto id = storage->login(user_name, password, address, external_authenticators); + auto id = storage->login(user_name, password, address, external_authenticators, false); std::lock_guard lock{mutex}; ids_cache.set(id, storage); return id; From d54b6c954241b9f92fc5c0bb36f24d2d08d86188 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Sun, 4 Oct 2020 20:32:08 +0000 Subject: [PATCH 098/441] add question in comments --- src/Storages/MergeTree/IMergeTreeDataPart.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index e36d1ee5b08..78daf6c9017 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -60,6 +60,7 @@ public: using Type = MergeTreeDataPartType; + IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index aece4b26130..da0c335c8e4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1182,7 +1182,10 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: { /// ALEXELEX /// is this a valid format or an old one? - /// should we support it for quorum parallel inserts? + /// discovered that minimalistic-format is new and is the default. + /// should we support old format for quorum parallel inserts? + /// if yes, let's use blocks/.../status ? hope no, it's too many if-else otherwise :) + /// but if so, how about this? ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( @@ -5733,11 +5736,10 @@ void StorageReplicatedMergeTree::getCommitPartOps( replica_path + "/parts/" + part->name, part_str, zkutil::CreateMode::Persistent)); - LOG_DEBUG(log, "ReplicatedMergeTreePartHeader::..: {}", part_str); } else { - /// ALEXELEXA + /// ALEXELEX /// also. should we support it? ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, From 2b28fde87f5b54cfc3539881ddb647211bcd9f85 Mon Sep 17 00:00:00 2001 From: r1j1k Date: Mon, 5 Oct 2020 09:04:22 +0300 Subject: [PATCH 099/441] Update type-conversion-functions.md --- .../functions/type-conversion-functions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 29b164545fc..f3ae7c85cf4 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -11,7 +11,7 @@ When you convert a value from one to another data type, you should remember that ClickHouse has the [same behavior as C++ programs](https://en.cppreference.com/w/cpp/language/implicit_conversion). -## toInt(8\|16\|32\|64\|128\|256) {#toint8163264} +## toInt(8\|16\|32\|64\|128\|256) {#toint8163264128256} Converts an input value to the [Int](../../sql-reference/data-types/int-uint.md) data type. This function family includes: @@ -62,7 +62,7 @@ select toInt64OrZero('123123'), toInt8OrZero('123qwe123') └─────────────────────────┴───────────────────────────┘ ``` -## toInt(8\|16\|32\|64\|128\|256)OrNull {#toint8163264ornull} +## toInt(8\|16\|32\|64\|128\|256)OrNull {#toint8163264128256ornull} It takes an argument of type String and tries to parse it into Int (8 \| 16 \| 32 \| 64 \| 128 \| 256). If failed, returns NULL. @@ -78,7 +78,7 @@ select toInt64OrNull('123123'), toInt8OrNull('123qwe123') └─────────────────────────┴───────────────────────────┘ ``` -## toUInt(8\|16\|32\|64\|256) {#touint8163264} +## toUInt(8\|16\|32\|64\|256) {#touint8163264256} Converts an input value to the [UInt](../../sql-reference/data-types/int-uint.md) data type. This function family includes: @@ -112,9 +112,9 @@ SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) └─────────────────────┴───────────────┴────────────────┴──────────────┘ ``` -## toUInt(8\|16\|32\|64\|256)OrZero {#touint8163264orzero} +## toUInt(8\|16\|32\|64\|256)OrZero {#touint8163264256orzero} -## toUInt(8\|16\|32\|64\|256)OrNull {#touint8163264ornull} +## toUInt(8\|16\|32\|64\|256)OrNull {#touint8163264256ornull} ## toFloat(32\|64) {#tofloat3264} @@ -134,7 +134,7 @@ SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8) ## toDateTimeOrNull {#todatetimeornull} -## toDecimal(32\|64\|128\|256) {#todecimal3264128} +## toDecimal(32\|64\|128\|256) {#todecimal3264128256} Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) data type with precision of `S`. The `value` can be a number or a string. The `S` (scale) parameter specifies the number of decimal places. @@ -143,7 +143,7 @@ Converts `value` to the [Decimal](../../sql-reference/data-types/decimal.md) dat - `toDecimal128(value, S)` - `toDecimal256(value, S)` -## toDecimal(32\|64\|128\|256)OrNull {#todecimal3264128ornull} +## toDecimal(32\|64\|128\|256)OrNull {#todecimal3264128256ornull} Converts an input string to a [Nullable(Decimal(P,S))](../../sql-reference/data-types/decimal.md) data type value. This family of functions include: @@ -188,7 +188,7 @@ SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val) └──────┴────────────────────────────────────────────────────┘ ``` -## toDecimal(32\|64\|128\|256)OrZero {#todecimal3264128orzero} +## toDecimal(32\|64\|128\|256)OrZero {#todecimal3264128256orzero} Converts an input value to the [Decimal(P,S)](../../sql-reference/data-types/decimal.md) data type. This family of functions include: From 0662432c5ce5e43d81c92a1aae6530408470a71d Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Mon, 5 Oct 2020 10:41:49 +0000 Subject: [PATCH 100/441] style check --- .../MergeTree/ReplicatedMergeTreeBlockEntry.h | 22 +++++++++---------- .../ReplicatedMergeTreeQuorumStatusEntry.h | 8 +++---- src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h index 97496538b16..b75b45cafba 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h @@ -20,8 +20,8 @@ namespace DB */ struct ReplicatedMergeTreeBlockEntry { - String part_name; - std::optional quorum_status; + String part_name; + std::optional quorum_status; ReplicatedMergeTreeBlockEntry() {} ReplicatedMergeTreeBlockEntry(const String & str) @@ -33,22 +33,22 @@ struct ReplicatedMergeTreeBlockEntry { out << part_name; - if (quorum_status) { + if (quorum_status) { out << "\n"; - quorum_status->writeText(out); + quorum_status->writeText(out); } } void readText(ReadBuffer & in) { - in >> part_name; + in >> part_name; - if (!in.eof()) - { - in >> "\n"; - quorum_status = ReplicatedMergeTreeQuorumStatusEntry(); - quorum_status->readText(in); - } + if (!in.eof()) + { + in >> "\n"; + quorum_status = ReplicatedMergeTreeQuorumStatusEntry(); + quorum_status->readText(in); + } } String toString() const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h index 95987ea13b1..bbf74808c2e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h @@ -67,10 +67,10 @@ struct ReplicatedMergeTreeQuorumStatusEntry readText(in); } - bool isQuorumReached() - { - return required_number_of_replicas && required_number_of_replicas <= replicas.size(); - } + bool isQuorumReached() + { + return required_number_of_replicas && required_number_of_replicas <= replicas.size(); + } }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index da0c335c8e4..6fbb2c588f0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3140,7 +3140,8 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st /// We update the node, registering there one more replica. code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); } - else { + else + { /// Update parallel quorum. It also might be reached here. code = zookeeper->trySet(quorum_parallel_status_path, block_entry.toString(), stat.version); } From aa391a47eb2cb0c93a899104d8c42653f669409f Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Mon, 5 Oct 2020 12:59:07 +0000 Subject: [PATCH 101/441] style check fix --- src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h index b75b45cafba..7d561c7fc6f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h @@ -33,7 +33,8 @@ struct ReplicatedMergeTreeBlockEntry { out << part_name; - if (quorum_status) { + if (quorum_status) + { out << "\n"; quorum_status->writeText(out); } From 537a497904c597cd39b017b7c8f64c713d862fe8 Mon Sep 17 00:00:00 2001 From: yulu86 Date: Mon, 5 Oct 2020 23:29:53 +0800 Subject: [PATCH 102/441] optimize Chinese tutorial --- docs/zh/getting-started/tutorial.md | 1328 +++++++++++++-------------- 1 file changed, 664 insertions(+), 664 deletions(-) diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index 2a82911cce4..afde4c31d23 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -1,664 +1,664 @@ ---- -toc_priority: 12 -toc_title: "\u6559\u7A0B" ---- - -# 点击教程 {#clickhouse-tutorial} - -## 从本教程中可以期待什么? {#what-to-expect-from-this-tutorial} - -通过本教程,您将学习如何设置一个简单的ClickHouse集群。 它会很小,但却是容错和可扩展的。 然后,我们将使用其中一个示例数据集来填充数据并执行一些演示查询。 - -## 单节点设置 {#single-node-setup} - -为了推迟分布式环境的复杂性,我们将首先在单个服务器或虚拟机上部署ClickHouse。 ClickHouse通常是从[deb](install.md#install-from-deb-packages) 或 [rpm](install.md#from-rpm-packages) 包安装,但对于不支持它们的操作系统也有 [替代方法](install.md#from-docker-image) 。 - -例如,您选择了从 `deb` 包安装,执行: - -``` bash -{% include 'install/deb.sh' %} -``` - -在我们安装的软件中包含这些包: - -- `clickhouse-client` 包,包含 [clickhouse-client](../interfaces/cli.md) 应用程序,它是交互式ClickHouse控制台客户端。 -- `clickhouse-common` 包,包含一个ClickHouse可执行文件。 -- `clickhouse-server` 包,包含要作为服务端运行的ClickHouse配置文件。 - -服务端配置文件位于 `/etc/clickhouse-server/`。 在进一步讨论之前,请注意 `config.xml`文件中的`` 元素. Path决定了数据存储的位置,因此该位置应该位于磁盘容量较大的卷上;默认值为 `/var/lib/clickhouse/`。 如果你想调整配置,考虑到它可能会在未来的软件包更新中被重写,直接编辑`config.xml` 文件并不方便。 推荐的方法是在[配置文件](../operations/configuration-files.md)目录创建文件,作为config.xml文件的“补丁”,用以复写配置元素。 - -你可能已经注意到了, `clickhouse-server` 安装后不会自动启动。 它也不会在更新后自动重新启动。 您启动服务端的方式取决于您的初始系统,通常情况下是这样: - -``` bash -sudo service clickhouse-server start -``` - -或 - -``` bash -sudo /etc/init.d/clickhouse-server start -``` - -服务端日志的默认位置是 `/var/log/clickhouse-server/`。当服务端在日志中记录 `Ready for connections` 消息,即表示服务端已准备好处理客户端连接。 - -一旦 `clickhouse-server` 启动并运行,我们可以利用 `clickhouse-client` 连接到服务端,并运行一些测试查询,如 `SELECT "Hello, world!";`. - -
- -Clickhouse-client的快速提示 - -交互模式: - -``` bash -clickhouse-client -clickhouse-client --host=... --port=... --user=... --password=... -``` - -启用多行查询: - -``` bash -clickhouse-client -m -clickhouse-client --multiline -``` - -以批处理模式运行查询: - -``` bash -clickhouse-client --query='SELECT 1' -echo 'SELECT 1' | clickhouse-client -clickhouse-client <<< 'SELECT 1' -``` - -从指定格式的文件中插入数据: - -``` bash -clickhouse-client --query='INSERT INTO table VALUES' < data.txt -clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv -``` - -
- -## 导入示例数据集 {#import-sample-dataset} - -现在是时候用一些示例数据填充我们的ClickHouse服务端。 在本教程中,我们将使用Yandex.Metrica的匿名数据,它是在ClickHouse成为开源之前作为生产环境运行的第一个服务(关于这一点的更多内容请参阅[ClickHouse历史](../introduction/history.md))。有 [多种导入Yandex.Metrica数据集的的方法](example-datasets/metrica.md),为了本教程,我们将使用最现实的一个。 - -### 下载并提取表数据 {#download-and-extract-table-data} - -``` bash -curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -``` - -提取的文件大小约为10GB。 - -### 创建表 {#create-tables} - -与大多数数据库管理系统一样,ClickHouse在逻辑上将表分组为数据库。包含一个 `default` 数据库,但我们将创建一个新的数据库 `tutorial`: - -``` bash -clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" -``` - -与创建数据库相比,创建表的语法要复杂得多(请参阅 [参考资料](../sql-reference/statements/create.md). 一般 `CREATE TABLE` 声明必须指定三个关键的事情: - -1. 要创建的表的名称。 -2. 表结构,例如:列名和对应的[数据类型](../sql-reference/data-types/index.md)。 -3. [表引擎](../engines/table-engines/index.md) 及其设置,这决定了对此表的查询操作是如何在物理层面执行的所有细节。 - -Yandex.Metrica是一个网络分析服务,样本数据集不包括其全部功能,因此只有两个表可以创建: - -- `hits` 表包含所有用户在服务所涵盖的所有网站上完成的每个操作。 -- `visits` 表包含预先构建的会话,而不是单个操作。 - -让我们看看并执行这些表的实际创建表查询: - -``` sql -CREATE TABLE tutorial.hits_v1 -( - `WatchID` UInt64, - `JavaEnable` UInt8, - `Title` String, - `GoodEvent` Int16, - `EventTime` DateTime, - `EventDate` Date, - `CounterID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RegionID` UInt32, - `UserID` UInt64, - `CounterClass` Int8, - `OS` UInt8, - `UserAgent` UInt8, - `URL` String, - `Referer` String, - `URLDomain` String, - `RefererDomain` String, - `Refresh` UInt8, - `IsRobot` UInt8, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `FlashMinor2` String, - `NetMajor` UInt8, - `NetMinor` UInt8, - `UserAgentMajor` UInt16, - `UserAgentMinor` FixedString(2), - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `MobilePhone` UInt8, - `MobilePhoneModel` String, - `Params` String, - `IPNetworkID` UInt32, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `IsArtifical` UInt8, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `ClientTimeZone` Int16, - `ClientEventTime` DateTime, - `SilverlightVersion1` UInt8, - `SilverlightVersion2` UInt8, - `SilverlightVersion3` UInt32, - `SilverlightVersion4` UInt16, - `PageCharset` String, - `CodeVersion` UInt32, - `IsLink` UInt8, - `IsDownload` UInt8, - `IsNotBounce` UInt8, - `FUniqID` UInt64, - `HID` UInt32, - `IsOldCounter` UInt8, - `IsEvent` UInt8, - `IsParameter` UInt8, - `DontCountHits` UInt8, - `WithHash` UInt8, - `HitColor` FixedString(1), - `UTCEventTime` DateTime, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `WindowName` Int32, - `OpenerName` Int32, - `HistoryLength` Int16, - `BrowserLanguage` FixedString(2), - `BrowserCountry` FixedString(2), - `SocialNetwork` String, - `SocialAction` String, - `HTTPError` UInt16, - `SendTiming` Int32, - `DNSTiming` Int32, - `ConnectTiming` Int32, - `ResponseStartTiming` Int32, - `ResponseEndTiming` Int32, - `FetchTiming` Int32, - `RedirectTiming` Int32, - `DOMInteractiveTiming` Int32, - `DOMContentLoadedTiming` Int32, - `DOMCompleteTiming` Int32, - `LoadEventStartTiming` Int32, - `LoadEventEndTiming` Int32, - `NSToDOMContentLoadedTiming` Int32, - `FirstPaintTiming` Int32, - `RedirectCount` Int8, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `ParamPrice` Int64, - `ParamOrderID` String, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `GoalsReached` Array(UInt32), - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `RefererHash` UInt64, - `URLHash` UInt64, - `CLID` UInt32, - `YCLID` UInt64, - `ShareService` String, - `ShareURL` String, - `ShareTitle` String, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `IslandID` FixedString(16), - `RequestNum` UInt32, - `RequestTry` UInt8 -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(EventDate) -ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -``` sql -CREATE TABLE tutorial.visits_v1 -( - `CounterID` UInt32, - `StartDate` Date, - `Sign` Int8, - `IsNew` UInt8, - `VisitID` UInt64, - `UserID` UInt64, - `StartTime` DateTime, - `Duration` UInt32, - `UTCStartTime` DateTime, - `PageViews` Int32, - `Hits` Int32, - `IsBounce` UInt8, - `Referer` String, - `StartURL` String, - `RefererDomain` String, - `StartURLDomain` String, - `EndURL` String, - `LinkURL` String, - `IsDownload` UInt8, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `PlaceID` Int32, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `IsYandex` UInt8, - `GoalReachesDepth` Int32, - `GoalReachesURL` Int32, - `GoalReachesAny` Int32, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `MobilePhoneModel` String, - `ClientEventTime` DateTime, - `RegionID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `IPNetworkID` UInt32, - `SilverlightVersion3` UInt32, - `CodeVersion` UInt32, - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `UserAgentMajor` UInt16, - `UserAgentMinor` UInt16, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `SilverlightVersion2` UInt8, - `SilverlightVersion4` UInt16, - `FlashVersion3` UInt16, - `FlashVersion4` UInt16, - `ClientTimeZone` Int16, - `OS` UInt8, - `UserAgent` UInt8, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `NetMajor` UInt8, - `NetMinor` UInt8, - `MobilePhone` UInt8, - `SilverlightVersion1` UInt8, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `JavaEnable` UInt8, - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `BrowserLanguage` UInt16, - `BrowserCountry` UInt16, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `Params` Array(String), - `Goals` Nested( - ID UInt32, - Serial UInt32, - EventTime DateTime, - Price Int64, - OrderID String, - CurrencyID UInt32), - `WatchIDs` Array(UInt64), - `ParamSumPrice` Int64, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `ClickLogID` UInt64, - `ClickEventID` Int32, - `ClickGoodEvent` Int32, - `ClickEventTime` DateTime, - `ClickPriorityID` Int32, - `ClickPhraseID` Int32, - `ClickPageID` Int32, - `ClickPlaceID` Int32, - `ClickTypeID` Int32, - `ClickResourceID` Int32, - `ClickCost` UInt32, - `ClickClientIP` UInt32, - `ClickDomainID` UInt32, - `ClickURL` String, - `ClickAttempt` UInt8, - `ClickOrderID` UInt32, - `ClickBannerID` UInt32, - `ClickMarketCategoryID` UInt32, - `ClickMarketPP` UInt32, - `ClickMarketCategoryName` String, - `ClickMarketPPName` String, - `ClickAWAPSCampaignName` String, - `ClickPageName` String, - `ClickTargetType` UInt16, - `ClickTargetPhraseID` UInt64, - `ClickContextType` UInt8, - `ClickSelectType` Int8, - `ClickOptions` String, - `ClickGroupBannerID` Int32, - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `FirstVisit` DateTime, - `PredLastVisit` Date, - `LastVisit` Date, - `TotalVisits` UInt32, - `TraficSource` Nested( - ID Int8, - SearchEngineID UInt16, - AdvEngineID UInt8, - PlaceID UInt16, - SocialSourceNetworkID UInt8, - Domain String, - SearchPhrase String, - SocialSourcePage String), - `Attendance` FixedString(16), - `CLID` UInt32, - `YCLID` UInt64, - `NormalizedRefererHash` UInt64, - `SearchPhraseHash` UInt64, - `RefererDomainHash` UInt64, - `NormalizedStartURLHash` UInt64, - `StartURLDomainHash` UInt64, - `NormalizedEndURLHash` UInt64, - `TopLevelDomain` UInt64, - `URLScheme` UInt64, - `OpenstatServiceNameHash` UInt64, - `OpenstatCampaignIDHash` UInt64, - `OpenstatAdIDHash` UInt64, - `OpenstatSourceIDHash` UInt64, - `UTMSourceHash` UInt64, - `UTMMediumHash` UInt64, - `UTMCampaignHash` UInt64, - `UTMContentHash` UInt64, - `UTMTermHash` UInt64, - `FromHash` UInt64, - `WebVisorEnabled` UInt8, - `WebVisorActivity` UInt32, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `Market` Nested( - Type UInt8, - GoalID UInt32, - OrderID String, - OrderPrice Int64, - PP UInt32, - DirectPlaceID UInt32, - DirectOrderID UInt32, - DirectBannerID UInt32, - GoodID String, - GoodName String, - GoodQuantity Int32, - GoodPrice Int64), - `IslandID` FixedString(16) -) -ENGINE = CollapsingMergeTree(Sign) -PARTITION BY toYYYYMM(StartDate) -ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -您可以使用`clickhouse-client`的交互模式执行这些查询(只需在终端中启动它,而不需要提前指定查询)。或者如果你愿意,可以尝试一些[替代接口](../interfaces/index.md)。 - -正如我们所看到的, `hits_v1` 使用 [基本的MergeTree引擎](../engines/table-engines/mergetree-family/mergetree.md),而 `visits_v1` 使用 [折叠树](../engines/table-engines/mergetree-family/collapsingmergetree.md) 变体。 - -### 导入数据 {#import-data} - -数据导入到ClickHouse是通过以下方式完成的 [INSERT INTO](../sql-reference/statements/insert-into.md) 查询像许多其他SQL数据库。 然而,数据通常是在一个提供 [支持的序列化格式](../interfaces/formats.md) 而不是 `VALUES` 子句(也支持)。 - -我们之前下载的文件是以制表符分隔的格式,所以这里是如何通过控制台客户端导入它们: - -``` bash -clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv -clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv -``` - -ClickHouse有很多 [要调整的设置](../operations/settings/index.md) 在控制台客户端中指定它们的一种方法是通过参数,我们可以看到 `--max_insert_block_size`. 找出可用的设置,它们意味着什么以及默认值的最简单方法是查询 `system.settings` 表: - -``` sql -SELECT name, value, changed, description -FROM system.settings -WHERE name LIKE '%max_insert_b%' -FORMAT TSV - -max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." -``` - -您也可以 [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) 导入后的表。 使用MergeTree-family引擎配置的表总是在后台合并数据部分以优化数据存储(或至少检查是否有意义)。 这些查询强制表引擎立即进行存储优化,而不是稍后进行一段时间: - -``` bash -clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" -clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" -``` - -这些查询开始一个I/O和CPU密集型操作,所以如果表一直接收到新数据,最好不要管它,让合并在后台运行。 - -现在我们可以检查表导入是否成功: - -``` bash -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" -``` - -## 查询示例 {#example-queries} - -``` sql -SELECT - StartURL AS URL, - AVG(Duration) AS AvgDuration -FROM tutorial.visits_v1 -WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' -GROUP BY URL -ORDER BY AvgDuration DESC -LIMIT 10 -``` - -``` sql -SELECT - sum(Sign) AS visits, - sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, - (100. * goal_visits) / visits AS goal_percent -FROM tutorial.visits_v1 -WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') -``` - -## 集群部署 {#cluster-deployment} - -ClickHouse集群是一个同质集群。 设置步骤: - -1. 在群集的所有计算机上安装ClickHouse服务器 -2. 在配置文件中设置群集配置 -3. 在每个实例上创建本地表 -4. 创建一个 [分布式表](../engines/table-engines/special/distributed.md) - -[分布式表](../engines/table-engines/special/distributed.md) 实际上是一种 “view” 到ClickHouse集群的本地表。 从分布式表中选择查询使用集群所有分片的资源执行。 您可以为多个集群指定configs,并创建多个分布式表,为不同的集群提供视图。 - -具有三个分片的集群的示例配置,每个分片一个副本: - -``` xml - - - - - example-perftest01j.yandex.ru - 9000 - - - - - example-perftest02j.yandex.ru - 9000 - - - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -为了进一步演示,让我们创建一个新的本地表 `CREATE TABLE` 我们用于查询 `hits_v1`,但不同的表名: - -``` sql -CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... -``` - -创建提供集群本地表视图的分布式表: - -``` sql -CREATE TABLE tutorial.hits_all AS tutorial.hits_local -ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); -``` - -常见的做法是在集群的所有计算机上创建类似的分布式表。 它允许在群集的任何计算机上运行分布式查询。 还有一个替代选项可以使用以下方法为给定的SELECT查询创建临时分布式表 [远程](../sql-reference/table-functions/remote.md) 表功能。 - -我们走吧 [INSERT SELECT](../sql-reference/statements/insert-into.md) 将该表传播到多个服务器。 - -``` sql -INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; -``` - -!!! warning "碌莽禄Notice:" - 这种方法不适合大型表的分片。 有一个单独的工具 [ツ环板-ョツ嘉ッツ偲](../operations/utilities/clickhouse-copier.md) 这可以重新分片任意大表。 - -正如您所期望的那样,如果计算量大的查询使用3台服务器而不是一个,则运行速度快N倍。 - -在这种情况下,我们使用了具有3个分片的集群,每个分片都包含一个副本。 - -为了在生产环境中提供弹性,我们建议每个分片应包含分布在多个可用区或数据中心(或至少机架)之间的2-3个副本。 请注意,ClickHouse支持无限数量的副本。 - -包含三个副本的一个分片集群的示例配置: - -``` xml - - ... - - - - example-perftest01j.yandex.ru - 9000 - - - example-perftest02j.yandex.ru - 9000 - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -启用本机复制 [动物园管理员](http://zookeeper.apache.org/) 是必需的。 ClickHouse负责所有副本的数据一致性,并在失败后自动运行恢复过程。 建议将ZooKeeper集群部署在单独的服务器上(其中没有其他进程,包括ClickHouse正在运行)。 - -!!! note "注" - ZooKeeper不是一个严格的requirement:在某些简单的情况下,您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是 **不** 建议,在这种情况下,ClickHouse将无法保证所有副本上的数据一致性。 因此,它成为您的应用程序的责任。 - -ZooKeeper位置在配置文件中指定: - -``` xml - - - zoo01.yandex.ru - 2181 - - - zoo02.yandex.ru - 2181 - - - zoo03.yandex.ru - 2181 - - -``` - -此外,我们需要设置宏来识别每个用于创建表的分片和副本: - -``` xml - - 01 - 01 - -``` - -如果在创建复制表时没有副本,则会实例化新的第一个副本。 如果已有实时副本,则新副本将克隆现有副本中的数据。 您可以选择首先创建所有复制的表,然后向其中插入数据。 另一种选择是创建一些副本,并在数据插入之后或期间添加其他副本。 - -``` sql -CREATE TABLE tutorial.hits_replica (...) -ENGINE = ReplcatedMergeTree( - '/clickhouse_perftest/tables/{shard}/hits', - '{replica}' -) -... -``` - -在这里,我们使用 [ReplicatedMergeTree](../engines/table-engines/mergetree-family/replication.md) 表引擎。 在参数中,我们指定包含分片和副本标识符的ZooKeeper路径。 - -``` sql -INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; -``` - -复制在多主机模式下运行。 数据可以加载到任何副本中,然后系统会自动将其与其他实例同步。 复制是异步的,因此在给定时刻,并非所有副本都可能包含最近插入的数据。 至少应有一个副本允许数据摄取。 其他人将同步数据和修复一致性,一旦他们将再次变得活跃。 请注意,这种方法允许最近插入的数据丢失的可能性很低。 - -[原始文章](https://clickhouse.tech/docs/en/getting_started/tutorial/) +--- +toc_priority: 12 +toc_title: "\u6559\u7A0B" +--- + +# 点击教程 {#clickhouse-tutorial} + +## 从本教程中可以期待什么? {#what-to-expect-from-this-tutorial} + +通过本教程,您将学习如何设置一个简单的ClickHouse集群。 它会很小,但却是容错和可扩展的。 然后,我们将使用其中一个示例数据集来填充数据并执行一些演示查询。 + +## 单节点设置 {#single-node-setup} + +为了推迟分布式环境的复杂性,我们将首先在单个服务器或虚拟机上部署ClickHouse。 ClickHouse通常是从[deb](install.md#install-from-deb-packages) 或 [rpm](install.md#from-rpm-packages) 包安装,但对于不支持它们的操作系统也有 [替代方法](install.md#from-docker-image) 。 + +例如,您选择了从 `deb` 包安装,执行: + +``` bash +{% include 'install/deb.sh' %} +``` + +在我们安装的软件中包含这些包: + +- `clickhouse-client` 包,包含 [clickhouse-client](../interfaces/cli.md) 应用程序,它是交互式ClickHouse控制台客户端。 +- `clickhouse-common` 包,包含一个ClickHouse可执行文件。 +- `clickhouse-server` 包,包含要作为服务端运行的ClickHouse配置文件。 + +服务端配置文件位于 `/etc/clickhouse-server/`。 在进一步讨论之前,请注意 `config.xml`文件中的`` 元素. Path决定了数据存储的位置,因此该位置应该位于磁盘容量较大的卷上;默认值为 `/var/lib/clickhouse/`。 如果你想调整配置,考虑到它可能会在未来的软件包更新中被重写,直接编辑`config.xml` 文件并不方便。 推荐的方法是在[配置文件](../operations/configuration-files.md)目录创建文件,作为config.xml文件的“补丁”,用以复写配置元素。 + +你可能已经注意到了, `clickhouse-server` 安装后不会自动启动。 它也不会在更新后自动重新启动。 您启动服务端的方式取决于您的初始系统,通常情况下是这样: + +``` bash +sudo service clickhouse-server start +``` + +或 + +``` bash +sudo /etc/init.d/clickhouse-server start +``` + +服务端日志的默认位置是 `/var/log/clickhouse-server/`。当服务端在日志中记录 `Ready for connections` 消息,即表示服务端已准备好处理客户端连接。 + +一旦 `clickhouse-server` 启动并运行,我们可以利用 `clickhouse-client` 连接到服务端,并运行一些测试查询,如 `SELECT "Hello, world!";`. + +
+ +Clickhouse-client的快速提示 + +交互模式: + +``` bash +clickhouse-client +clickhouse-client --host=... --port=... --user=... --password=... +``` + +启用多行查询: + +``` bash +clickhouse-client -m +clickhouse-client --multiline +``` + +以批处理模式运行查询: + +``` bash +clickhouse-client --query='SELECT 1' +echo 'SELECT 1' | clickhouse-client +clickhouse-client <<< 'SELECT 1' +``` + +从指定格式的文件中插入数据: + +``` bash +clickhouse-client --query='INSERT INTO table VALUES' < data.txt +clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv +``` + +
+ +## 导入示例数据集 {#import-sample-dataset} + +现在是时候用一些示例数据填充我们的ClickHouse服务端。 在本教程中,我们将使用Yandex.Metrica的匿名数据,它是在ClickHouse成为开源之前作为生产环境运行的第一个服务(关于这一点的更多内容请参阅[ClickHouse历史](../introduction/history.md))。有 [多种导入Yandex.Metrica数据集的的方法](example-datasets/metrica.md),为了本教程,我们将使用最现实的一个。 + +### 下载并提取表数据 {#download-and-extract-table-data} + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +``` + +提取的文件大小约为10GB。 + +### 创建表 {#create-tables} + +与大多数数据库管理系统一样,ClickHouse在逻辑上将表分组为数据库。包含一个 `default` 数据库,但我们将创建一个新的数据库 `tutorial`: + +``` bash +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" +``` + +与创建数据库相比,创建表的语法要复杂得多(请参阅 [参考资料](../sql-reference/statements/create.md). 一般 `CREATE TABLE` 声明必须指定三个关键的事情: + +1. 要创建的表的名称。 +2. 表结构,例如:列名和对应的[数据类型](../sql-reference/data-types/index.md)。 +3. [表引擎](../engines/table-engines/index.md) 及其设置,这决定了对此表的查询操作是如何在物理层面执行的所有细节。 + +Yandex.Metrica是一个网络分析服务,样本数据集不包括其全部功能,因此只有两个表可以创建: + +- `hits` 表包含所有用户在服务所涵盖的所有网站上完成的每个操作。 +- `visits` 表包含预先构建的会话,而不是单个操作。 + +让我们看看并执行这些表的实际创建表查询: + +``` sql +CREATE TABLE tutorial.hits_v1 +( + `WatchID` UInt64, + `JavaEnable` UInt8, + `Title` String, + `GoodEvent` Int16, + `EventTime` DateTime, + `EventDate` Date, + `CounterID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RegionID` UInt32, + `UserID` UInt64, + `CounterClass` Int8, + `OS` UInt8, + `UserAgent` UInt8, + `URL` String, + `Referer` String, + `URLDomain` String, + `RefererDomain` String, + `Refresh` UInt8, + `IsRobot` UInt8, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `FlashMinor2` String, + `NetMajor` UInt8, + `NetMinor` UInt8, + `UserAgentMajor` UInt16, + `UserAgentMinor` FixedString(2), + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `MobilePhone` UInt8, + `MobilePhoneModel` String, + `Params` String, + `IPNetworkID` UInt32, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `IsArtifical` UInt8, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `ClientTimeZone` Int16, + `ClientEventTime` DateTime, + `SilverlightVersion1` UInt8, + `SilverlightVersion2` UInt8, + `SilverlightVersion3` UInt32, + `SilverlightVersion4` UInt16, + `PageCharset` String, + `CodeVersion` UInt32, + `IsLink` UInt8, + `IsDownload` UInt8, + `IsNotBounce` UInt8, + `FUniqID` UInt64, + `HID` UInt32, + `IsOldCounter` UInt8, + `IsEvent` UInt8, + `IsParameter` UInt8, + `DontCountHits` UInt8, + `WithHash` UInt8, + `HitColor` FixedString(1), + `UTCEventTime` DateTime, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `WindowName` Int32, + `OpenerName` Int32, + `HistoryLength` Int16, + `BrowserLanguage` FixedString(2), + `BrowserCountry` FixedString(2), + `SocialNetwork` String, + `SocialAction` String, + `HTTPError` UInt16, + `SendTiming` Int32, + `DNSTiming` Int32, + `ConnectTiming` Int32, + `ResponseStartTiming` Int32, + `ResponseEndTiming` Int32, + `FetchTiming` Int32, + `RedirectTiming` Int32, + `DOMInteractiveTiming` Int32, + `DOMContentLoadedTiming` Int32, + `DOMCompleteTiming` Int32, + `LoadEventStartTiming` Int32, + `LoadEventEndTiming` Int32, + `NSToDOMContentLoadedTiming` Int32, + `FirstPaintTiming` Int32, + `RedirectCount` Int8, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `ParamPrice` Int64, + `ParamOrderID` String, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `GoalsReached` Array(UInt32), + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `RefererHash` UInt64, + `URLHash` UInt64, + `CLID` UInt32, + `YCLID` UInt64, + `ShareService` String, + `ShareURL` String, + `ShareTitle` String, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `IslandID` FixedString(16), + `RequestNum` UInt32, + `RequestTry` UInt8 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID)) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +``` sql +CREATE TABLE tutorial.visits_v1 +( + `CounterID` UInt32, + `StartDate` Date, + `Sign` Int8, + `IsNew` UInt8, + `VisitID` UInt64, + `UserID` UInt64, + `StartTime` DateTime, + `Duration` UInt32, + `UTCStartTime` DateTime, + `PageViews` Int32, + `Hits` Int32, + `IsBounce` UInt8, + `Referer` String, + `StartURL` String, + `RefererDomain` String, + `StartURLDomain` String, + `EndURL` String, + `LinkURL` String, + `IsDownload` UInt8, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `PlaceID` Int32, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `IsYandex` UInt8, + `GoalReachesDepth` Int32, + `GoalReachesURL` Int32, + `GoalReachesAny` Int32, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `MobilePhoneModel` String, + `ClientEventTime` DateTime, + `RegionID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `IPNetworkID` UInt32, + `SilverlightVersion3` UInt32, + `CodeVersion` UInt32, + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `UserAgentMajor` UInt16, + `UserAgentMinor` UInt16, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `SilverlightVersion2` UInt8, + `SilverlightVersion4` UInt16, + `FlashVersion3` UInt16, + `FlashVersion4` UInt16, + `ClientTimeZone` Int16, + `OS` UInt8, + `UserAgent` UInt8, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `NetMajor` UInt8, + `NetMinor` UInt8, + `MobilePhone` UInt8, + `SilverlightVersion1` UInt8, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `JavaEnable` UInt8, + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `BrowserLanguage` UInt16, + `BrowserCountry` UInt16, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `Params` Array(String), + `Goals` Nested( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32), + `WatchIDs` Array(UInt64), + `ParamSumPrice` Int64, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `ClickLogID` UInt64, + `ClickEventID` Int32, + `ClickGoodEvent` Int32, + `ClickEventTime` DateTime, + `ClickPriorityID` Int32, + `ClickPhraseID` Int32, + `ClickPageID` Int32, + `ClickPlaceID` Int32, + `ClickTypeID` Int32, + `ClickResourceID` Int32, + `ClickCost` UInt32, + `ClickClientIP` UInt32, + `ClickDomainID` UInt32, + `ClickURL` String, + `ClickAttempt` UInt8, + `ClickOrderID` UInt32, + `ClickBannerID` UInt32, + `ClickMarketCategoryID` UInt32, + `ClickMarketPP` UInt32, + `ClickMarketCategoryName` String, + `ClickMarketPPName` String, + `ClickAWAPSCampaignName` String, + `ClickPageName` String, + `ClickTargetType` UInt16, + `ClickTargetPhraseID` UInt64, + `ClickContextType` UInt8, + `ClickSelectType` Int8, + `ClickOptions` String, + `ClickGroupBannerID` Int32, + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `FirstVisit` DateTime, + `PredLastVisit` Date, + `LastVisit` Date, + `TotalVisits` UInt32, + `TraficSource` Nested( + ID Int8, + SearchEngineID UInt16, + AdvEngineID UInt8, + PlaceID UInt16, + SocialSourceNetworkID UInt8, + Domain String, + SearchPhrase String, + SocialSourcePage String), + `Attendance` FixedString(16), + `CLID` UInt32, + `YCLID` UInt64, + `NormalizedRefererHash` UInt64, + `SearchPhraseHash` UInt64, + `RefererDomainHash` UInt64, + `NormalizedStartURLHash` UInt64, + `StartURLDomainHash` UInt64, + `NormalizedEndURLHash` UInt64, + `TopLevelDomain` UInt64, + `URLScheme` UInt64, + `OpenstatServiceNameHash` UInt64, + `OpenstatCampaignIDHash` UInt64, + `OpenstatAdIDHash` UInt64, + `OpenstatSourceIDHash` UInt64, + `UTMSourceHash` UInt64, + `UTMMediumHash` UInt64, + `UTMCampaignHash` UInt64, + `UTMContentHash` UInt64, + `UTMTermHash` UInt64, + `FromHash` UInt64, + `WebVisorEnabled` UInt8, + `WebVisorActivity` UInt32, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `Market` Nested( + Type UInt8, + GoalID UInt32, + OrderID String, + OrderPrice Int64, + PP UInt32, + DirectPlaceID UInt32, + DirectOrderID UInt32, + DirectBannerID UInt32, + GoodID String, + GoodName String, + GoodQuantity Int32, + GoodPrice Int64), + `IslandID` FixedString(16) +) +ENGINE = CollapsingMergeTree(Sign) +PARTITION BY toYYYYMM(StartDate) +ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +您可以使用`clickhouse-client`的交互模式执行这些查询(只需在终端中启动它,而不需要提前指定查询)。或者如果你愿意,可以尝试一些[替代接口](../interfaces/index.md)。 + +正如我们所看到的, `hits_v1` 使用 [基本的MergeTree引擎](../engines/table-engines/mergetree-family/mergetree.md),而 `visits_v1` 使用 [折叠树](../engines/table-engines/mergetree-family/collapsingmergetree.md) 变体。 + +### 导入数据 {#import-data} + +数据导入到ClickHouse是通过以下方式完成的 [INSERT INTO](../sql-reference/statements/insert-into.md) 查询像许多其他SQL数据库。 然而,数据通常是在一个提供 [支持的序列化格式](../interfaces/formats.md) 而不是 `VALUES` 子句(也支持)。 + +我们之前下载的文件是以制表符分隔的格式,所以这里是如何通过控制台客户端导入它们: + +``` bash +clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv +clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv +``` + +ClickHouse有很多 [要调整的设置](../operations/settings/index.md) 在控制台客户端中指定它们的一种方法是通过参数,就像我们看到上面语句中的 `--max_insert_block_size`。找出可用的设置、含义及其默认值的最简单方法是查询 `system.settings` 表: + +``` sql +SELECT name, value, changed, description +FROM system.settings +WHERE name LIKE '%max_insert_b%' +FORMAT TSV + +max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." +``` + +您也可以 [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) 导入后的表。 使用MergeTree-family引擎配置的表总是在后台合并数据部分以优化数据存储(或至少检查是否有意义)。 这些查询强制表引擎立即进行存储优化,而不是稍后一段时间执行: + +``` bash +clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" +clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" +``` + +这些查询开始一个I/O和CPU密集型操作,所以如果表一直接收到新数据,最好不要管它,让合并在后台运行。 + +现在我们可以检查表导入是否成功: + +``` bash +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" +``` + +## 查询示例 {#example-queries} + +``` sql +SELECT + StartURL AS URL, + AVG(Duration) AS AvgDuration +FROM tutorial.visits_v1 +WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' +GROUP BY URL +ORDER BY AvgDuration DESC +LIMIT 10 +``` + +``` sql +SELECT + sum(Sign) AS visits, + sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, + (100. * goal_visits) / visits AS goal_percent +FROM tutorial.visits_v1 +WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') +``` + +## 集群部署 {#cluster-deployment} + +ClickHouse集群是一个同质集群。 设置步骤: + +1. 在群集的所有机器上安装ClickHouse服务端 +2. 在配置文件中设置群集配置 +3. 在每个实例上创建本地表 +4. 创建一个 [分布式表](../engines/table-engines/special/distributed.md) + +[分布式表](../engines/table-engines/special/distributed.md) 实际上是一种 “视图”,映射到ClickHouse集群的本地表。 从分布式表中执行 **SELECT** 查询会使用集群所有分片的资源。您可以为多个集群指定configs,并创建多个分布式表,为不同的集群提供视图。 + +具有三个分片,每个分片一个副本的集群的示例配置: + +``` xml + + + + + example-perftest01j.yandex.ru + 9000 + + + + + example-perftest02j.yandex.ru + 9000 + + + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +为了进一步演示,让我们使用和创建 `hits_v1` 表相同的 `CREATE TABLE` 语句创建一个新的本地表,但表名不同: + +``` sql +CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... +``` + +创建提供集群本地表视图的分布式表: + +``` sql +CREATE TABLE tutorial.hits_all AS tutorial.hits_local +ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); +``` + +常见的做法是在集群的所有计算机上创建类似的分布式表。 它允许在群集的任何计算机上运行分布式查询。 还有一个替代选项可以使用以下方法为给定的SELECT查询创建临时分布式表 [远程](../sql-reference/table-functions/remote.md) 表功能。 + +让我们运行 [INSERT SELECT](../sql-reference/statements/insert-into.md) 将该表传播到多个服务器。 + +``` sql +INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; +``` + +!!! warning "注意:" + 这种方法不适合大型表的分片。 有一个单独的工具 [clickhouse-copier](../operations/utilities/clickhouse-copier.md) 这可以重新分片任意大表。 + +正如您所期望的那样,如果计算量大的查询使用3台服务器而不是一个,则运行速度快N倍。 + +在这种情况下,我们使用了具有3个分片的集群,每个分片都包含一个副本。 + +为了在生产环境中提供弹性,我们建议每个分片应包含分布在多个可用区或数据中心(或至少机架)之间的2-3个副本。 请注意,ClickHouse支持无限数量的副本。 + +包含三个副本的一个分片集群的示例配置: + +``` xml + + ... + + + + example-perftest01j.yandex.ru + 9000 + + + example-perftest02j.yandex.ru + 9000 + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +启用本机复制 [Zookeeper](http://zookeeper.apache.org/) 是必需的。 ClickHouse负责所有副本的数据一致性,并在失败后自动运行恢复过程。 建议将ZooKeeper集群部署在单独的服务器上(其中没有其他进程,包括运行的ClickHouse)。 + +!!! note "注" + ZooKeeper不是一个严格的要求:在某些简单的情况下,您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是 **不** 建议的,在这种情况下,ClickHouse将无法保证所有副本上的数据一致性。 因此,由您的应用来保证这一点。 + +ZooKeeper位置在配置文件中指定: + +``` xml + + + zoo01.yandex.ru + 2181 + + + zoo02.yandex.ru + 2181 + + + zoo03.yandex.ru + 2181 + + +``` + +此外,我们需要设置宏来识别每个用于创建表的分片和副本: + +``` xml + + 01 + 01 + +``` + +如果在创建复制表时没有副本,则会实例化新的第一个副本。 如果已有实时副本,则新副本将克隆现有副本中的数据。 您可以选择首先创建所有复制的表,然后向其中插入数据。 另一种选择是创建一些副本,并在数据插入之后或期间添加其他副本。 + +``` sql +CREATE TABLE tutorial.hits_replica (...) +ENGINE = ReplcatedMergeTree( + '/clickhouse_perftest/tables/{shard}/hits', + '{replica}' +) +... +``` + +在这里,我们使用 [ReplicatedMergeTree](../engines/table-engines/mergetree-family/replication.md) 表引擎。 在参数中,我们指定包含分片和副本标识符的ZooKeeper路径。 + +``` sql +INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; +``` + +复制在多主机模式下运行。 数据可以加载到任何副本中,然后系统会自动将其与其他实例同步。 复制是异步的,因此在给定时刻,并非所有副本都可能包含最近插入的数据。 至少应有一个副本允许数据摄取。 其他人将同步数据和修复一致性,一旦他们将再次变得活跃。 请注意,这种方法允许最近插入的数据丢失的可能性很低。 + +[原始文章](https://clickhouse.tech/docs/en/getting_started/tutorial/) From 791862d416d6ab6034c77fffb78bc808ee61ed91 Mon Sep 17 00:00:00 2001 From: yulu86 Date: Mon, 5 Oct 2020 23:54:44 +0800 Subject: [PATCH 103/441] optimize Chinese tutorial --- docs/zh/getting-started/tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index afde4c31d23..384512c8a60 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -612,7 +612,7 @@ INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; 启用本机复制 [Zookeeper](http://zookeeper.apache.org/) 是必需的。 ClickHouse负责所有副本的数据一致性,并在失败后自动运行恢复过程。 建议将ZooKeeper集群部署在单独的服务器上(其中没有其他进程,包括运行的ClickHouse)。 !!! note "注" - ZooKeeper不是一个严格的要求:在某些简单的情况下,您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是 **不** 建议的,在这种情况下,ClickHouse将无法保证所有副本上的数据一致性。 因此,由您的应用来保证这一点。 + ZooKeeper不是一个严格的要求:在某些简单的情况下,您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是 **不** 建议的,在这种情况下,ClickHouse将无法保证所有副本上的数据一致性。 因此需要由您的应用来保证这一点。 ZooKeeper位置在配置文件中指定: From 0cc7f3eac8f99c6e5bc0ee828dcda8c244827537 Mon Sep 17 00:00:00 2001 From: yulu86 Date: Tue, 6 Oct 2020 00:05:34 +0800 Subject: [PATCH 104/441] optimize Chinese tutorial --- docs/zh/getting-started/tutorial.md | 1328 +++++++++++++-------------- 1 file changed, 664 insertions(+), 664 deletions(-) diff --git a/docs/zh/getting-started/tutorial.md b/docs/zh/getting-started/tutorial.md index 384512c8a60..07f595b4354 100644 --- a/docs/zh/getting-started/tutorial.md +++ b/docs/zh/getting-started/tutorial.md @@ -1,664 +1,664 @@ ---- -toc_priority: 12 -toc_title: "\u6559\u7A0B" ---- - -# 点击教程 {#clickhouse-tutorial} - -## 从本教程中可以期待什么? {#what-to-expect-from-this-tutorial} - -通过本教程,您将学习如何设置一个简单的ClickHouse集群。 它会很小,但却是容错和可扩展的。 然后,我们将使用其中一个示例数据集来填充数据并执行一些演示查询。 - -## 单节点设置 {#single-node-setup} - -为了推迟分布式环境的复杂性,我们将首先在单个服务器或虚拟机上部署ClickHouse。 ClickHouse通常是从[deb](install.md#install-from-deb-packages) 或 [rpm](install.md#from-rpm-packages) 包安装,但对于不支持它们的操作系统也有 [替代方法](install.md#from-docker-image) 。 - -例如,您选择了从 `deb` 包安装,执行: - -``` bash -{% include 'install/deb.sh' %} -``` - -在我们安装的软件中包含这些包: - -- `clickhouse-client` 包,包含 [clickhouse-client](../interfaces/cli.md) 应用程序,它是交互式ClickHouse控制台客户端。 -- `clickhouse-common` 包,包含一个ClickHouse可执行文件。 -- `clickhouse-server` 包,包含要作为服务端运行的ClickHouse配置文件。 - -服务端配置文件位于 `/etc/clickhouse-server/`。 在进一步讨论之前,请注意 `config.xml`文件中的`` 元素. Path决定了数据存储的位置,因此该位置应该位于磁盘容量较大的卷上;默认值为 `/var/lib/clickhouse/`。 如果你想调整配置,考虑到它可能会在未来的软件包更新中被重写,直接编辑`config.xml` 文件并不方便。 推荐的方法是在[配置文件](../operations/configuration-files.md)目录创建文件,作为config.xml文件的“补丁”,用以复写配置元素。 - -你可能已经注意到了, `clickhouse-server` 安装后不会自动启动。 它也不会在更新后自动重新启动。 您启动服务端的方式取决于您的初始系统,通常情况下是这样: - -``` bash -sudo service clickhouse-server start -``` - -或 - -``` bash -sudo /etc/init.d/clickhouse-server start -``` - -服务端日志的默认位置是 `/var/log/clickhouse-server/`。当服务端在日志中记录 `Ready for connections` 消息,即表示服务端已准备好处理客户端连接。 - -一旦 `clickhouse-server` 启动并运行,我们可以利用 `clickhouse-client` 连接到服务端,并运行一些测试查询,如 `SELECT "Hello, world!";`. - -
- -Clickhouse-client的快速提示 - -交互模式: - -``` bash -clickhouse-client -clickhouse-client --host=... --port=... --user=... --password=... -``` - -启用多行查询: - -``` bash -clickhouse-client -m -clickhouse-client --multiline -``` - -以批处理模式运行查询: - -``` bash -clickhouse-client --query='SELECT 1' -echo 'SELECT 1' | clickhouse-client -clickhouse-client <<< 'SELECT 1' -``` - -从指定格式的文件中插入数据: - -``` bash -clickhouse-client --query='INSERT INTO table VALUES' < data.txt -clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv -``` - -
- -## 导入示例数据集 {#import-sample-dataset} - -现在是时候用一些示例数据填充我们的ClickHouse服务端。 在本教程中,我们将使用Yandex.Metrica的匿名数据,它是在ClickHouse成为开源之前作为生产环境运行的第一个服务(关于这一点的更多内容请参阅[ClickHouse历史](../introduction/history.md))。有 [多种导入Yandex.Metrica数据集的的方法](example-datasets/metrica.md),为了本教程,我们将使用最现实的一个。 - -### 下载并提取表数据 {#download-and-extract-table-data} - -``` bash -curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv -curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv -``` - -提取的文件大小约为10GB。 - -### 创建表 {#create-tables} - -与大多数数据库管理系统一样,ClickHouse在逻辑上将表分组为数据库。包含一个 `default` 数据库,但我们将创建一个新的数据库 `tutorial`: - -``` bash -clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" -``` - -与创建数据库相比,创建表的语法要复杂得多(请参阅 [参考资料](../sql-reference/statements/create.md). 一般 `CREATE TABLE` 声明必须指定三个关键的事情: - -1. 要创建的表的名称。 -2. 表结构,例如:列名和对应的[数据类型](../sql-reference/data-types/index.md)。 -3. [表引擎](../engines/table-engines/index.md) 及其设置,这决定了对此表的查询操作是如何在物理层面执行的所有细节。 - -Yandex.Metrica是一个网络分析服务,样本数据集不包括其全部功能,因此只有两个表可以创建: - -- `hits` 表包含所有用户在服务所涵盖的所有网站上完成的每个操作。 -- `visits` 表包含预先构建的会话,而不是单个操作。 - -让我们看看并执行这些表的实际创建表查询: - -``` sql -CREATE TABLE tutorial.hits_v1 -( - `WatchID` UInt64, - `JavaEnable` UInt8, - `Title` String, - `GoodEvent` Int16, - `EventTime` DateTime, - `EventDate` Date, - `CounterID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RegionID` UInt32, - `UserID` UInt64, - `CounterClass` Int8, - `OS` UInt8, - `UserAgent` UInt8, - `URL` String, - `Referer` String, - `URLDomain` String, - `RefererDomain` String, - `Refresh` UInt8, - `IsRobot` UInt8, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `FlashMinor2` String, - `NetMajor` UInt8, - `NetMinor` UInt8, - `UserAgentMajor` UInt16, - `UserAgentMinor` FixedString(2), - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `MobilePhone` UInt8, - `MobilePhoneModel` String, - `Params` String, - `IPNetworkID` UInt32, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `IsArtifical` UInt8, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `ClientTimeZone` Int16, - `ClientEventTime` DateTime, - `SilverlightVersion1` UInt8, - `SilverlightVersion2` UInt8, - `SilverlightVersion3` UInt32, - `SilverlightVersion4` UInt16, - `PageCharset` String, - `CodeVersion` UInt32, - `IsLink` UInt8, - `IsDownload` UInt8, - `IsNotBounce` UInt8, - `FUniqID` UInt64, - `HID` UInt32, - `IsOldCounter` UInt8, - `IsEvent` UInt8, - `IsParameter` UInt8, - `DontCountHits` UInt8, - `WithHash` UInt8, - `HitColor` FixedString(1), - `UTCEventTime` DateTime, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `WindowName` Int32, - `OpenerName` Int32, - `HistoryLength` Int16, - `BrowserLanguage` FixedString(2), - `BrowserCountry` FixedString(2), - `SocialNetwork` String, - `SocialAction` String, - `HTTPError` UInt16, - `SendTiming` Int32, - `DNSTiming` Int32, - `ConnectTiming` Int32, - `ResponseStartTiming` Int32, - `ResponseEndTiming` Int32, - `FetchTiming` Int32, - `RedirectTiming` Int32, - `DOMInteractiveTiming` Int32, - `DOMContentLoadedTiming` Int32, - `DOMCompleteTiming` Int32, - `LoadEventStartTiming` Int32, - `LoadEventEndTiming` Int32, - `NSToDOMContentLoadedTiming` Int32, - `FirstPaintTiming` Int32, - `RedirectCount` Int8, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `ParamPrice` Int64, - `ParamOrderID` String, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `GoalsReached` Array(UInt32), - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `RefererHash` UInt64, - `URLHash` UInt64, - `CLID` UInt32, - `YCLID` UInt64, - `ShareService` String, - `ShareURL` String, - `ShareTitle` String, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `IslandID` FixedString(16), - `RequestNum` UInt32, - `RequestTry` UInt8 -) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(EventDate) -ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -``` sql -CREATE TABLE tutorial.visits_v1 -( - `CounterID` UInt32, - `StartDate` Date, - `Sign` Int8, - `IsNew` UInt8, - `VisitID` UInt64, - `UserID` UInt64, - `StartTime` DateTime, - `Duration` UInt32, - `UTCStartTime` DateTime, - `PageViews` Int32, - `Hits` Int32, - `IsBounce` UInt8, - `Referer` String, - `StartURL` String, - `RefererDomain` String, - `StartURLDomain` String, - `EndURL` String, - `LinkURL` String, - `IsDownload` UInt8, - `TraficSourceID` Int8, - `SearchEngineID` UInt16, - `SearchPhrase` String, - `AdvEngineID` UInt8, - `PlaceID` Int32, - `RefererCategories` Array(UInt16), - `URLCategories` Array(UInt16), - `URLRegions` Array(UInt32), - `RefererRegions` Array(UInt32), - `IsYandex` UInt8, - `GoalReachesDepth` Int32, - `GoalReachesURL` Int32, - `GoalReachesAny` Int32, - `SocialSourceNetworkID` UInt8, - `SocialSourcePage` String, - `MobilePhoneModel` String, - `ClientEventTime` DateTime, - `RegionID` UInt32, - `ClientIP` UInt32, - `ClientIP6` FixedString(16), - `RemoteIP` UInt32, - `RemoteIP6` FixedString(16), - `IPNetworkID` UInt32, - `SilverlightVersion3` UInt32, - `CodeVersion` UInt32, - `ResolutionWidth` UInt16, - `ResolutionHeight` UInt16, - `UserAgentMajor` UInt16, - `UserAgentMinor` UInt16, - `WindowClientWidth` UInt16, - `WindowClientHeight` UInt16, - `SilverlightVersion2` UInt8, - `SilverlightVersion4` UInt16, - `FlashVersion3` UInt16, - `FlashVersion4` UInt16, - `ClientTimeZone` Int16, - `OS` UInt8, - `UserAgent` UInt8, - `ResolutionDepth` UInt8, - `FlashMajor` UInt8, - `FlashMinor` UInt8, - `NetMajor` UInt8, - `NetMinor` UInt8, - `MobilePhone` UInt8, - `SilverlightVersion1` UInt8, - `Age` UInt8, - `Sex` UInt8, - `Income` UInt8, - `JavaEnable` UInt8, - `CookieEnable` UInt8, - `JavascriptEnable` UInt8, - `IsMobile` UInt8, - `BrowserLanguage` UInt16, - `BrowserCountry` UInt16, - `Interests` UInt16, - `Robotness` UInt8, - `GeneralInterests` Array(UInt16), - `Params` Array(String), - `Goals` Nested( - ID UInt32, - Serial UInt32, - EventTime DateTime, - Price Int64, - OrderID String, - CurrencyID UInt32), - `WatchIDs` Array(UInt64), - `ParamSumPrice` Int64, - `ParamCurrency` FixedString(3), - `ParamCurrencyID` UInt16, - `ClickLogID` UInt64, - `ClickEventID` Int32, - `ClickGoodEvent` Int32, - `ClickEventTime` DateTime, - `ClickPriorityID` Int32, - `ClickPhraseID` Int32, - `ClickPageID` Int32, - `ClickPlaceID` Int32, - `ClickTypeID` Int32, - `ClickResourceID` Int32, - `ClickCost` UInt32, - `ClickClientIP` UInt32, - `ClickDomainID` UInt32, - `ClickURL` String, - `ClickAttempt` UInt8, - `ClickOrderID` UInt32, - `ClickBannerID` UInt32, - `ClickMarketCategoryID` UInt32, - `ClickMarketPP` UInt32, - `ClickMarketCategoryName` String, - `ClickMarketPPName` String, - `ClickAWAPSCampaignName` String, - `ClickPageName` String, - `ClickTargetType` UInt16, - `ClickTargetPhraseID` UInt64, - `ClickContextType` UInt8, - `ClickSelectType` Int8, - `ClickOptions` String, - `ClickGroupBannerID` Int32, - `OpenstatServiceName` String, - `OpenstatCampaignID` String, - `OpenstatAdID` String, - `OpenstatSourceID` String, - `UTMSource` String, - `UTMMedium` String, - `UTMCampaign` String, - `UTMContent` String, - `UTMTerm` String, - `FromTag` String, - `HasGCLID` UInt8, - `FirstVisit` DateTime, - `PredLastVisit` Date, - `LastVisit` Date, - `TotalVisits` UInt32, - `TraficSource` Nested( - ID Int8, - SearchEngineID UInt16, - AdvEngineID UInt8, - PlaceID UInt16, - SocialSourceNetworkID UInt8, - Domain String, - SearchPhrase String, - SocialSourcePage String), - `Attendance` FixedString(16), - `CLID` UInt32, - `YCLID` UInt64, - `NormalizedRefererHash` UInt64, - `SearchPhraseHash` UInt64, - `RefererDomainHash` UInt64, - `NormalizedStartURLHash` UInt64, - `StartURLDomainHash` UInt64, - `NormalizedEndURLHash` UInt64, - `TopLevelDomain` UInt64, - `URLScheme` UInt64, - `OpenstatServiceNameHash` UInt64, - `OpenstatCampaignIDHash` UInt64, - `OpenstatAdIDHash` UInt64, - `OpenstatSourceIDHash` UInt64, - `UTMSourceHash` UInt64, - `UTMMediumHash` UInt64, - `UTMCampaignHash` UInt64, - `UTMContentHash` UInt64, - `UTMTermHash` UInt64, - `FromHash` UInt64, - `WebVisorEnabled` UInt8, - `WebVisorActivity` UInt32, - `ParsedParams` Nested( - Key1 String, - Key2 String, - Key3 String, - Key4 String, - Key5 String, - ValueDouble Float64), - `Market` Nested( - Type UInt8, - GoalID UInt32, - OrderID String, - OrderPrice Int64, - PP UInt32, - DirectPlaceID UInt32, - DirectOrderID UInt32, - DirectBannerID UInt32, - GoodID String, - GoodName String, - GoodQuantity Int32, - GoodPrice Int64), - `IslandID` FixedString(16) -) -ENGINE = CollapsingMergeTree(Sign) -PARTITION BY toYYYYMM(StartDate) -ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) -SAMPLE BY intHash32(UserID) -SETTINGS index_granularity = 8192 -``` - -您可以使用`clickhouse-client`的交互模式执行这些查询(只需在终端中启动它,而不需要提前指定查询)。或者如果你愿意,可以尝试一些[替代接口](../interfaces/index.md)。 - -正如我们所看到的, `hits_v1` 使用 [基本的MergeTree引擎](../engines/table-engines/mergetree-family/mergetree.md),而 `visits_v1` 使用 [折叠树](../engines/table-engines/mergetree-family/collapsingmergetree.md) 变体。 - -### 导入数据 {#import-data} - -数据导入到ClickHouse是通过以下方式完成的 [INSERT INTO](../sql-reference/statements/insert-into.md) 查询像许多其他SQL数据库。 然而,数据通常是在一个提供 [支持的序列化格式](../interfaces/formats.md) 而不是 `VALUES` 子句(也支持)。 - -我们之前下载的文件是以制表符分隔的格式,所以这里是如何通过控制台客户端导入它们: - -``` bash -clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv -clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv -``` - -ClickHouse有很多 [要调整的设置](../operations/settings/index.md) 在控制台客户端中指定它们的一种方法是通过参数,就像我们看到上面语句中的 `--max_insert_block_size`。找出可用的设置、含义及其默认值的最简单方法是查询 `system.settings` 表: - -``` sql -SELECT name, value, changed, description -FROM system.settings -WHERE name LIKE '%max_insert_b%' -FORMAT TSV - -max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." -``` - -您也可以 [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) 导入后的表。 使用MergeTree-family引擎配置的表总是在后台合并数据部分以优化数据存储(或至少检查是否有意义)。 这些查询强制表引擎立即进行存储优化,而不是稍后一段时间执行: - -``` bash -clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" -clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" -``` - -这些查询开始一个I/O和CPU密集型操作,所以如果表一直接收到新数据,最好不要管它,让合并在后台运行。 - -现在我们可以检查表导入是否成功: - -``` bash -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" -clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" -``` - -## 查询示例 {#example-queries} - -``` sql -SELECT - StartURL AS URL, - AVG(Duration) AS AvgDuration -FROM tutorial.visits_v1 -WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' -GROUP BY URL -ORDER BY AvgDuration DESC -LIMIT 10 -``` - -``` sql -SELECT - sum(Sign) AS visits, - sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, - (100. * goal_visits) / visits AS goal_percent -FROM tutorial.visits_v1 -WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') -``` - -## 集群部署 {#cluster-deployment} - -ClickHouse集群是一个同质集群。 设置步骤: - -1. 在群集的所有机器上安装ClickHouse服务端 -2. 在配置文件中设置群集配置 -3. 在每个实例上创建本地表 -4. 创建一个 [分布式表](../engines/table-engines/special/distributed.md) - -[分布式表](../engines/table-engines/special/distributed.md) 实际上是一种 “视图”,映射到ClickHouse集群的本地表。 从分布式表中执行 **SELECT** 查询会使用集群所有分片的资源。您可以为多个集群指定configs,并创建多个分布式表,为不同的集群提供视图。 - -具有三个分片,每个分片一个副本的集群的示例配置: - -``` xml - - - - - example-perftest01j.yandex.ru - 9000 - - - - - example-perftest02j.yandex.ru - 9000 - - - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -为了进一步演示,让我们使用和创建 `hits_v1` 表相同的 `CREATE TABLE` 语句创建一个新的本地表,但表名不同: - -``` sql -CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... -``` - -创建提供集群本地表视图的分布式表: - -``` sql -CREATE TABLE tutorial.hits_all AS tutorial.hits_local -ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); -``` - -常见的做法是在集群的所有计算机上创建类似的分布式表。 它允许在群集的任何计算机上运行分布式查询。 还有一个替代选项可以使用以下方法为给定的SELECT查询创建临时分布式表 [远程](../sql-reference/table-functions/remote.md) 表功能。 - -让我们运行 [INSERT SELECT](../sql-reference/statements/insert-into.md) 将该表传播到多个服务器。 - -``` sql -INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; -``` - -!!! warning "注意:" - 这种方法不适合大型表的分片。 有一个单独的工具 [clickhouse-copier](../operations/utilities/clickhouse-copier.md) 这可以重新分片任意大表。 - -正如您所期望的那样,如果计算量大的查询使用3台服务器而不是一个,则运行速度快N倍。 - -在这种情况下,我们使用了具有3个分片的集群,每个分片都包含一个副本。 - -为了在生产环境中提供弹性,我们建议每个分片应包含分布在多个可用区或数据中心(或至少机架)之间的2-3个副本。 请注意,ClickHouse支持无限数量的副本。 - -包含三个副本的一个分片集群的示例配置: - -``` xml - - ... - - - - example-perftest01j.yandex.ru - 9000 - - - example-perftest02j.yandex.ru - 9000 - - - example-perftest03j.yandex.ru - 9000 - - - - -``` - -启用本机复制 [Zookeeper](http://zookeeper.apache.org/) 是必需的。 ClickHouse负责所有副本的数据一致性,并在失败后自动运行恢复过程。 建议将ZooKeeper集群部署在单独的服务器上(其中没有其他进程,包括运行的ClickHouse)。 - -!!! note "注" - ZooKeeper不是一个严格的要求:在某些简单的情况下,您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是 **不** 建议的,在这种情况下,ClickHouse将无法保证所有副本上的数据一致性。 因此需要由您的应用来保证这一点。 - -ZooKeeper位置在配置文件中指定: - -``` xml - - - zoo01.yandex.ru - 2181 - - - zoo02.yandex.ru - 2181 - - - zoo03.yandex.ru - 2181 - - -``` - -此外,我们需要设置宏来识别每个用于创建表的分片和副本: - -``` xml - - 01 - 01 - -``` - -如果在创建复制表时没有副本,则会实例化新的第一个副本。 如果已有实时副本,则新副本将克隆现有副本中的数据。 您可以选择首先创建所有复制的表,然后向其中插入数据。 另一种选择是创建一些副本,并在数据插入之后或期间添加其他副本。 - -``` sql -CREATE TABLE tutorial.hits_replica (...) -ENGINE = ReplcatedMergeTree( - '/clickhouse_perftest/tables/{shard}/hits', - '{replica}' -) -... -``` - -在这里,我们使用 [ReplicatedMergeTree](../engines/table-engines/mergetree-family/replication.md) 表引擎。 在参数中,我们指定包含分片和副本标识符的ZooKeeper路径。 - -``` sql -INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; -``` - -复制在多主机模式下运行。 数据可以加载到任何副本中,然后系统会自动将其与其他实例同步。 复制是异步的,因此在给定时刻,并非所有副本都可能包含最近插入的数据。 至少应有一个副本允许数据摄取。 其他人将同步数据和修复一致性,一旦他们将再次变得活跃。 请注意,这种方法允许最近插入的数据丢失的可能性很低。 - -[原始文章](https://clickhouse.tech/docs/en/getting_started/tutorial/) +--- +toc_priority: 12 +toc_title: "\u6559\u7A0B" +--- + +# 点击教程 {#clickhouse-tutorial} + +## 从本教程中可以期待什么? {#what-to-expect-from-this-tutorial} + +通过本教程,您将学习如何设置一个简单的ClickHouse集群。 它会很小,但却是容错和可扩展的。 然后,我们将使用其中一个示例数据集来填充数据并执行一些演示查询。 + +## 单节点设置 {#single-node-setup} + +为了推迟分布式环境的复杂性,我们将首先在单个服务器或虚拟机上部署ClickHouse。 ClickHouse通常是从[deb](install.md#install-from-deb-packages) 或 [rpm](install.md#from-rpm-packages) 包安装,但对于不支持它们的操作系统也有 [替代方法](install.md#from-docker-image) 。 + +例如,您选择了从 `deb` 包安装,执行: + +``` bash +{% include 'install/deb.sh' %} +``` + +在我们安装的软件中包含这些包: + +- `clickhouse-client` 包,包含 [clickhouse-client](../interfaces/cli.md) 应用程序,它是交互式ClickHouse控制台客户端。 +- `clickhouse-common` 包,包含一个ClickHouse可执行文件。 +- `clickhouse-server` 包,包含要作为服务端运行的ClickHouse配置文件。 + +服务端配置文件位于 `/etc/clickhouse-server/`。 在进一步讨论之前,请注意 `config.xml`文件中的`` 元素. Path决定了数据存储的位置,因此该位置应该位于磁盘容量较大的卷上;默认值为 `/var/lib/clickhouse/`。 如果你想调整配置,考虑到它可能会在未来的软件包更新中被重写,直接编辑`config.xml` 文件并不方便。 推荐的方法是在[配置文件](../operations/configuration-files.md)目录创建文件,作为config.xml文件的“补丁”,用以复写配置元素。 + +你可能已经注意到了, `clickhouse-server` 安装后不会自动启动。 它也不会在更新后自动重新启动。 您启动服务端的方式取决于您的初始系统,通常情况下是这样: + +``` bash +sudo service clickhouse-server start +``` + +或 + +``` bash +sudo /etc/init.d/clickhouse-server start +``` + +服务端日志的默认位置是 `/var/log/clickhouse-server/`。当服务端在日志中记录 `Ready for connections` 消息,即表示服务端已准备好处理客户端连接。 + +一旦 `clickhouse-server` 启动并运行,我们可以利用 `clickhouse-client` 连接到服务端,并运行一些测试查询,如 `SELECT "Hello, world!";`. + +
+ +Clickhouse-client的快速提示 + +交互模式: + +``` bash +clickhouse-client +clickhouse-client --host=... --port=... --user=... --password=... +``` + +启用多行查询: + +``` bash +clickhouse-client -m +clickhouse-client --multiline +``` + +以批处理模式运行查询: + +``` bash +clickhouse-client --query='SELECT 1' +echo 'SELECT 1' | clickhouse-client +clickhouse-client <<< 'SELECT 1' +``` + +从指定格式的文件中插入数据: + +``` bash +clickhouse-client --query='INSERT INTO table VALUES' < data.txt +clickhouse-client --query='INSERT INTO table FORMAT TabSeparated' < data.tsv +``` + +
+ +## 导入示例数据集 {#import-sample-dataset} + +现在是时候用一些示例数据填充我们的ClickHouse服务端。 在本教程中,我们将使用Yandex.Metrica的匿名数据,它是在ClickHouse成为开源之前作为生产环境运行的第一个服务(关于这一点的更多内容请参阅[ClickHouse历史](../introduction/history.md))。有 [多种导入Yandex.Metrica数据集的的方法](example-datasets/metrica.md),为了本教程,我们将使用最现实的一个。 + +### 下载并提取表数据 {#download-and-extract-table-data} + +``` bash +curl https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_v1.tsv.xz | unxz --threads=`nproc` > hits_v1.tsv +curl https://clickhouse-datasets.s3.yandex.net/visits/tsv/visits_v1.tsv.xz | unxz --threads=`nproc` > visits_v1.tsv +``` + +提取的文件大小约为10GB。 + +### 创建表 {#create-tables} + +与大多数数据库管理系统一样,ClickHouse在逻辑上将表分组为数据库。包含一个 `default` 数据库,但我们将创建一个新的数据库 `tutorial`: + +``` bash +clickhouse-client --query "CREATE DATABASE IF NOT EXISTS tutorial" +``` + +与创建数据库相比,创建表的语法要复杂得多(请参阅 [参考资料](../sql-reference/statements/create.md). 一般 `CREATE TABLE` 声明必须指定三个关键的事情: + +1. 要创建的表的名称。 +2. 表结构,例如:列名和对应的[数据类型](../sql-reference/data-types/index.md)。 +3. [表引擎](../engines/table-engines/index.md) 及其设置,这决定了对此表的查询操作是如何在物理层面执行的所有细节。 + +Yandex.Metrica是一个网络分析服务,样本数据集不包括其全部功能,因此只有两个表可以创建: + +- `hits` 表包含所有用户在服务所涵盖的所有网站上完成的每个操作。 +- `visits` 表包含预先构建的会话,而不是单个操作。 + +让我们看看并执行这些表的实际创建表查询: + +``` sql +CREATE TABLE tutorial.hits_v1 +( + `WatchID` UInt64, + `JavaEnable` UInt8, + `Title` String, + `GoodEvent` Int16, + `EventTime` DateTime, + `EventDate` Date, + `CounterID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RegionID` UInt32, + `UserID` UInt64, + `CounterClass` Int8, + `OS` UInt8, + `UserAgent` UInt8, + `URL` String, + `Referer` String, + `URLDomain` String, + `RefererDomain` String, + `Refresh` UInt8, + `IsRobot` UInt8, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `FlashMinor2` String, + `NetMajor` UInt8, + `NetMinor` UInt8, + `UserAgentMajor` UInt16, + `UserAgentMinor` FixedString(2), + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `MobilePhone` UInt8, + `MobilePhoneModel` String, + `Params` String, + `IPNetworkID` UInt32, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `IsArtifical` UInt8, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `ClientTimeZone` Int16, + `ClientEventTime` DateTime, + `SilverlightVersion1` UInt8, + `SilverlightVersion2` UInt8, + `SilverlightVersion3` UInt32, + `SilverlightVersion4` UInt16, + `PageCharset` String, + `CodeVersion` UInt32, + `IsLink` UInt8, + `IsDownload` UInt8, + `IsNotBounce` UInt8, + `FUniqID` UInt64, + `HID` UInt32, + `IsOldCounter` UInt8, + `IsEvent` UInt8, + `IsParameter` UInt8, + `DontCountHits` UInt8, + `WithHash` UInt8, + `HitColor` FixedString(1), + `UTCEventTime` DateTime, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `WindowName` Int32, + `OpenerName` Int32, + `HistoryLength` Int16, + `BrowserLanguage` FixedString(2), + `BrowserCountry` FixedString(2), + `SocialNetwork` String, + `SocialAction` String, + `HTTPError` UInt16, + `SendTiming` Int32, + `DNSTiming` Int32, + `ConnectTiming` Int32, + `ResponseStartTiming` Int32, + `ResponseEndTiming` Int32, + `FetchTiming` Int32, + `RedirectTiming` Int32, + `DOMInteractiveTiming` Int32, + `DOMContentLoadedTiming` Int32, + `DOMCompleteTiming` Int32, + `LoadEventStartTiming` Int32, + `LoadEventEndTiming` Int32, + `NSToDOMContentLoadedTiming` Int32, + `FirstPaintTiming` Int32, + `RedirectCount` Int8, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `ParamPrice` Int64, + `ParamOrderID` String, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `GoalsReached` Array(UInt32), + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `RefererHash` UInt64, + `URLHash` UInt64, + `CLID` UInt32, + `YCLID` UInt64, + `ShareService` String, + `ShareURL` String, + `ShareTitle` String, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `IslandID` FixedString(16), + `RequestNum` UInt32, + `RequestTry` UInt8 +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(EventDate) +ORDER BY (CounterID, EventDate, intHash32(UserID)) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +``` sql +CREATE TABLE tutorial.visits_v1 +( + `CounterID` UInt32, + `StartDate` Date, + `Sign` Int8, + `IsNew` UInt8, + `VisitID` UInt64, + `UserID` UInt64, + `StartTime` DateTime, + `Duration` UInt32, + `UTCStartTime` DateTime, + `PageViews` Int32, + `Hits` Int32, + `IsBounce` UInt8, + `Referer` String, + `StartURL` String, + `RefererDomain` String, + `StartURLDomain` String, + `EndURL` String, + `LinkURL` String, + `IsDownload` UInt8, + `TraficSourceID` Int8, + `SearchEngineID` UInt16, + `SearchPhrase` String, + `AdvEngineID` UInt8, + `PlaceID` Int32, + `RefererCategories` Array(UInt16), + `URLCategories` Array(UInt16), + `URLRegions` Array(UInt32), + `RefererRegions` Array(UInt32), + `IsYandex` UInt8, + `GoalReachesDepth` Int32, + `GoalReachesURL` Int32, + `GoalReachesAny` Int32, + `SocialSourceNetworkID` UInt8, + `SocialSourcePage` String, + `MobilePhoneModel` String, + `ClientEventTime` DateTime, + `RegionID` UInt32, + `ClientIP` UInt32, + `ClientIP6` FixedString(16), + `RemoteIP` UInt32, + `RemoteIP6` FixedString(16), + `IPNetworkID` UInt32, + `SilverlightVersion3` UInt32, + `CodeVersion` UInt32, + `ResolutionWidth` UInt16, + `ResolutionHeight` UInt16, + `UserAgentMajor` UInt16, + `UserAgentMinor` UInt16, + `WindowClientWidth` UInt16, + `WindowClientHeight` UInt16, + `SilverlightVersion2` UInt8, + `SilverlightVersion4` UInt16, + `FlashVersion3` UInt16, + `FlashVersion4` UInt16, + `ClientTimeZone` Int16, + `OS` UInt8, + `UserAgent` UInt8, + `ResolutionDepth` UInt8, + `FlashMajor` UInt8, + `FlashMinor` UInt8, + `NetMajor` UInt8, + `NetMinor` UInt8, + `MobilePhone` UInt8, + `SilverlightVersion1` UInt8, + `Age` UInt8, + `Sex` UInt8, + `Income` UInt8, + `JavaEnable` UInt8, + `CookieEnable` UInt8, + `JavascriptEnable` UInt8, + `IsMobile` UInt8, + `BrowserLanguage` UInt16, + `BrowserCountry` UInt16, + `Interests` UInt16, + `Robotness` UInt8, + `GeneralInterests` Array(UInt16), + `Params` Array(String), + `Goals` Nested( + ID UInt32, + Serial UInt32, + EventTime DateTime, + Price Int64, + OrderID String, + CurrencyID UInt32), + `WatchIDs` Array(UInt64), + `ParamSumPrice` Int64, + `ParamCurrency` FixedString(3), + `ParamCurrencyID` UInt16, + `ClickLogID` UInt64, + `ClickEventID` Int32, + `ClickGoodEvent` Int32, + `ClickEventTime` DateTime, + `ClickPriorityID` Int32, + `ClickPhraseID` Int32, + `ClickPageID` Int32, + `ClickPlaceID` Int32, + `ClickTypeID` Int32, + `ClickResourceID` Int32, + `ClickCost` UInt32, + `ClickClientIP` UInt32, + `ClickDomainID` UInt32, + `ClickURL` String, + `ClickAttempt` UInt8, + `ClickOrderID` UInt32, + `ClickBannerID` UInt32, + `ClickMarketCategoryID` UInt32, + `ClickMarketPP` UInt32, + `ClickMarketCategoryName` String, + `ClickMarketPPName` String, + `ClickAWAPSCampaignName` String, + `ClickPageName` String, + `ClickTargetType` UInt16, + `ClickTargetPhraseID` UInt64, + `ClickContextType` UInt8, + `ClickSelectType` Int8, + `ClickOptions` String, + `ClickGroupBannerID` Int32, + `OpenstatServiceName` String, + `OpenstatCampaignID` String, + `OpenstatAdID` String, + `OpenstatSourceID` String, + `UTMSource` String, + `UTMMedium` String, + `UTMCampaign` String, + `UTMContent` String, + `UTMTerm` String, + `FromTag` String, + `HasGCLID` UInt8, + `FirstVisit` DateTime, + `PredLastVisit` Date, + `LastVisit` Date, + `TotalVisits` UInt32, + `TraficSource` Nested( + ID Int8, + SearchEngineID UInt16, + AdvEngineID UInt8, + PlaceID UInt16, + SocialSourceNetworkID UInt8, + Domain String, + SearchPhrase String, + SocialSourcePage String), + `Attendance` FixedString(16), + `CLID` UInt32, + `YCLID` UInt64, + `NormalizedRefererHash` UInt64, + `SearchPhraseHash` UInt64, + `RefererDomainHash` UInt64, + `NormalizedStartURLHash` UInt64, + `StartURLDomainHash` UInt64, + `NormalizedEndURLHash` UInt64, + `TopLevelDomain` UInt64, + `URLScheme` UInt64, + `OpenstatServiceNameHash` UInt64, + `OpenstatCampaignIDHash` UInt64, + `OpenstatAdIDHash` UInt64, + `OpenstatSourceIDHash` UInt64, + `UTMSourceHash` UInt64, + `UTMMediumHash` UInt64, + `UTMCampaignHash` UInt64, + `UTMContentHash` UInt64, + `UTMTermHash` UInt64, + `FromHash` UInt64, + `WebVisorEnabled` UInt8, + `WebVisorActivity` UInt32, + `ParsedParams` Nested( + Key1 String, + Key2 String, + Key3 String, + Key4 String, + Key5 String, + ValueDouble Float64), + `Market` Nested( + Type UInt8, + GoalID UInt32, + OrderID String, + OrderPrice Int64, + PP UInt32, + DirectPlaceID UInt32, + DirectOrderID UInt32, + DirectBannerID UInt32, + GoodID String, + GoodName String, + GoodQuantity Int32, + GoodPrice Int64), + `IslandID` FixedString(16) +) +ENGINE = CollapsingMergeTree(Sign) +PARTITION BY toYYYYMM(StartDate) +ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) +SAMPLE BY intHash32(UserID) +SETTINGS index_granularity = 8192 +``` + +您可以使用`clickhouse-client`的交互模式执行这些查询(只需在终端中启动它,而不需要提前指定查询)。或者如果你愿意,可以尝试一些[替代接口](../interfaces/index.md)。 + +正如我们所看到的, `hits_v1` 使用 [基本的MergeTree引擎](../engines/table-engines/mergetree-family/mergetree.md),而 `visits_v1` 使用 [折叠树](../engines/table-engines/mergetree-family/collapsingmergetree.md) 变体。 + +### 导入数据 {#import-data} + +数据导入到ClickHouse是通过以下方式完成的 [INSERT INTO](../sql-reference/statements/insert-into.md) 查询像许多其他SQL数据库。 然而,数据通常是在一个提供 [支持的序列化格式](../interfaces/formats.md) 而不是 `VALUES` 子句(也支持)。 + +我们之前下载的文件是以制表符分隔的格式,所以这里是如何通过控制台客户端导入它们: + +``` bash +clickhouse-client --query "INSERT INTO tutorial.hits_v1 FORMAT TSV" --max_insert_block_size=100000 < hits_v1.tsv +clickhouse-client --query "INSERT INTO tutorial.visits_v1 FORMAT TSV" --max_insert_block_size=100000 < visits_v1.tsv +``` + +ClickHouse有很多 [要调整的设置](../operations/settings/index.md) 在控制台客户端中指定它们的一种方法是通过参数,就像我们看到上面语句中的 `--max_insert_block_size`。找出可用的设置、含义及其默认值的最简单方法是查询 `system.settings` 表: + +``` sql +SELECT name, value, changed, description +FROM system.settings +WHERE name LIKE '%max_insert_b%' +FORMAT TSV + +max_insert_block_size 1048576 0 "The maximum block size for insertion, if we control the creation of blocks for insertion." +``` + +您也可以 [OPTIMIZE](../sql-reference/statements/misc.md#misc_operations-optimize) 导入后的表。 使用MergeTree-family引擎配置的表总是在后台合并数据部分以优化数据存储(或至少检查是否有意义)。 这些查询强制表引擎立即进行存储优化,而不是稍后一段时间执行: + +``` bash +clickhouse-client --query "OPTIMIZE TABLE tutorial.hits_v1 FINAL" +clickhouse-client --query "OPTIMIZE TABLE tutorial.visits_v1 FINAL" +``` + +这些查询开始一个I/O和CPU密集型操作,所以如果表一直接收到新数据,最好不要管它,让合并在后台运行。 + +现在我们可以检查表导入是否成功: + +``` bash +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.hits_v1" +clickhouse-client --query "SELECT COUNT(*) FROM tutorial.visits_v1" +``` + +## 查询示例 {#example-queries} + +``` sql +SELECT + StartURL AS URL, + AVG(Duration) AS AvgDuration +FROM tutorial.visits_v1 +WHERE StartDate BETWEEN '2014-03-23' AND '2014-03-30' +GROUP BY URL +ORDER BY AvgDuration DESC +LIMIT 10 +``` + +``` sql +SELECT + sum(Sign) AS visits, + sumIf(Sign, has(Goals.ID, 1105530)) AS goal_visits, + (100. * goal_visits) / visits AS goal_percent +FROM tutorial.visits_v1 +WHERE (CounterID = 912887) AND (toYYYYMM(StartDate) = 201403) AND (domain(StartURL) = 'yandex.ru') +``` + +## 集群部署 {#cluster-deployment} + +ClickHouse集群是一个同质集群。 设置步骤: + +1. 在群集的所有机器上安装ClickHouse服务端 +2. 在配置文件中设置群集配置 +3. 在每个实例上创建本地表 +4. 创建一个 [分布式表](../engines/table-engines/special/distributed.md) + +[分布式表](../engines/table-engines/special/distributed.md) 实际上是一种 “视图”,映射到ClickHouse集群的本地表。 从分布式表中执行 **SELECT** 查询会使用集群所有分片的资源。 您可以为多个集群指定configs,并创建多个分布式表,为不同的集群提供视图。 + +具有三个分片,每个分片一个副本的集群的示例配置: + +``` xml + + + + + example-perftest01j.yandex.ru + 9000 + + + + + example-perftest02j.yandex.ru + 9000 + + + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +为了进一步演示,让我们使用和创建 `hits_v1` 表相同的 `CREATE TABLE` 语句创建一个新的本地表,但表名不同: + +``` sql +CREATE TABLE tutorial.hits_local (...) ENGINE = MergeTree() ... +``` + +创建提供集群本地表视图的分布式表: + +``` sql +CREATE TABLE tutorial.hits_all AS tutorial.hits_local +ENGINE = Distributed(perftest_3shards_1replicas, tutorial, hits_local, rand()); +``` + +常见的做法是在集群的所有计算机上创建类似的分布式表。 它允许在群集的任何计算机上运行分布式查询。 还有一个替代选项可以使用以下方法为给定的SELECT查询创建临时分布式表 [远程](../sql-reference/table-functions/remote.md) 表功能。 + +让我们运行 [INSERT SELECT](../sql-reference/statements/insert-into.md) 将该表传播到多个服务器。 + +``` sql +INSERT INTO tutorial.hits_all SELECT * FROM tutorial.hits_v1; +``` + +!!! warning "注意:" + 这种方法不适合大型表的分片。 有一个单独的工具 [clickhouse-copier](../operations/utilities/clickhouse-copier.md) 这可以重新分片任意大表。 + +正如您所期望的那样,如果计算量大的查询使用3台服务器而不是一个,则运行速度快N倍。 + +在这种情况下,我们使用了具有3个分片的集群,每个分片都包含一个副本。 + +为了在生产环境中提供弹性,我们建议每个分片应包含分布在多个可用区或数据中心(或至少机架)之间的2-3个副本。 请注意,ClickHouse支持无限数量的副本。 + +包含三个副本的一个分片集群的示例配置: + +``` xml + + ... + + + + example-perftest01j.yandex.ru + 9000 + + + example-perftest02j.yandex.ru + 9000 + + + example-perftest03j.yandex.ru + 9000 + + + + +``` + +启用本机复制 [Zookeeper](http://zookeeper.apache.org/) 是必需的。 ClickHouse负责所有副本的数据一致性,并在失败后自动运行恢复过程。 建议将ZooKeeper集群部署在单独的服务器上(其中没有其他进程,包括运行的ClickHouse)。 + +!!! note "注" + ZooKeeper不是一个严格的要求:在某些简单的情况下,您可以通过将数据写入应用程序代码中的所有副本来复制数据。 这种方法是 **不** 建议的,在这种情况下,ClickHouse将无法保证所有副本上的数据一致性。 因此需要由您的应用来保证这一点。 + +ZooKeeper位置在配置文件中指定: + +``` xml + + + zoo01.yandex.ru + 2181 + + + zoo02.yandex.ru + 2181 + + + zoo03.yandex.ru + 2181 + + +``` + +此外,我们需要设置宏来识别每个用于创建表的分片和副本: + +``` xml + + 01 + 01 + +``` + +如果在创建复制表时没有副本,则会实例化新的第一个副本。 如果已有实时副本,则新副本将克隆现有副本中的数据。 您可以选择首先创建所有复制的表,然后向其中插入数据。 另一种选择是创建一些副本,并在数据插入之后或期间添加其他副本。 + +``` sql +CREATE TABLE tutorial.hits_replica (...) +ENGINE = ReplcatedMergeTree( + '/clickhouse_perftest/tables/{shard}/hits', + '{replica}' +) +... +``` + +在这里,我们使用 [ReplicatedMergeTree](../engines/table-engines/mergetree-family/replication.md) 表引擎。 在参数中,我们指定包含分片和副本标识符的ZooKeeper路径。 + +``` sql +INSERT INTO tutorial.hits_replica SELECT * FROM tutorial.hits_local; +``` + +复制在多主机模式下运行。 数据可以加载到任何副本中,然后系统会自动将其与其他实例同步。 复制是异步的,因此在给定时刻,并非所有副本都可能包含最近插入的数据。 至少应有一个副本允许数据摄取。 其他人将同步数据和修复一致性,一旦他们将再次变得活跃。 请注意,这种方法允许最近插入的数据丢失的可能性很低。 + +[原始文章](https://clickhouse.tech/docs/en/getting_started/tutorial/) From 2be62bbd1b72321c1a155a80cf30cc80733d1752 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Mon, 5 Oct 2020 16:58:26 +0000 Subject: [PATCH 105/441] fix test_00814 --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 128fcb66f96..c88172f0701 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1764,7 +1764,13 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( Strings partitions = zookeeper->getChildren(queue.replica_path + "/parts"); for (const String & partition : partitions) { - auto header = ReplicatedMergeTreePartHeader::fromString(zookeeper->get(queue.replica_path + "/parts/" + partition)); + auto part_str = zookeeper->get(queue.replica_path + "/parts/" + partition); + if (part_str.empty()) + { + /// use_minimalistic_part_header_in_zookeeper + continue; + } + auto header = ReplicatedMergeTreePartHeader::fromString(part_str); if (header.getBlockID()) { ReplicatedMergeTreeBlockEntry block(zookeeper->get(queue.zookeeper_path + "/blocks/" + *header.getBlockID())); From 41e0148d728e8853dc06bec9da1c00c0a306f09b Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Mon, 5 Oct 2020 20:30:50 +0000 Subject: [PATCH 106/441] build checker fix (style) and change comments --- .../MergeTree/ReplicatedMergeTreePartHeader.cpp | 10 +++++----- src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 10 ++-------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp index 932c4497d30..3f9c86d7c49 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp @@ -37,11 +37,11 @@ ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromColumnsAndCheck ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromColumnsChecksumsBlockID( const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums, - const String & block_id_) + const String & block_id_name) { MinimalisticDataPartChecksums checksums; checksums.computeTotalChecksums(full_checksums); - return ReplicatedMergeTreePartHeader(getSipHash(columns.toString()), std::move(checksums), block_id_); + return ReplicatedMergeTreePartHeader(getSipHash(columns.toString()), std::move(checksums), block_id_name); } void ReplicatedMergeTreePartHeader::read(ReadBuffer & in) @@ -52,9 +52,9 @@ void ReplicatedMergeTreePartHeader::read(ReadBuffer & in) if (!in.eof()) { - String block_id_; - in >> "\nblock_id: " >> block_id_; - block_id = block_id_; + String new_block_id; + in >> "\nblock_id: " >> new_block_id; + block_id = new_block_id; } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h index c67aba29f32..a5cca569b56 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h @@ -30,7 +30,7 @@ public: const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums); static ReplicatedMergeTreePartHeader fromColumnsChecksumsBlockID( - const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums, const String & block_id_); + const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums, const String & block_id_name); void read(ReadBuffer & in); static ReplicatedMergeTreePartHeader fromString(const String & str); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6fbb2c588f0..10b52ecc517 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1180,12 +1180,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - /// ALEXELEX - /// is this a valid format or an old one? - /// discovered that minimalistic-format is new and is the default. - /// should we support old format for quorum parallel inserts? - /// if yes, let's use blocks/.../status ? hope no, it's too many if-else otherwise :) - /// but if so, how about this? + /// alexelex: if (local_part_header.getBlockID()) throw or skip it? ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( @@ -5740,8 +5735,7 @@ void StorageReplicatedMergeTree::getCommitPartOps( } else { - /// ALEXELEX - /// also. should we support it? + /// alexelex: if (!block_id.empty()) throw or skip it? ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, "", From d98366ed79fcf99323ab8f05b8d984540b546760 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Tue, 6 Oct 2020 11:20:47 +0300 Subject: [PATCH 107/441] Better initialization of S3 storage. --- src/IO/S3/PocoHTTPClient.cpp | 21 +++++++++++++++++++++ src/IO/S3/PocoHTTPClient.h | 2 ++ src/IO/S3Common.cpp | 28 ++++++++-------------------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index c34d7719131..a649b76730b 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -17,6 +17,9 @@ #include #include #include +#include + +#include namespace ProfileEvents @@ -51,6 +54,24 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( { } +void PocoHTTPClientConfiguration::updateSchemeAndRegion() +{ + if (!endpointOverride.empty()) + { + static const RE2 region_pattern(R"(^s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); + Poco::URI uri(endpointOverride); + if (uri.getScheme() == "http") + scheme = Aws::Http::Scheme::HTTP; + + String matched_region; + if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, &matched_region)) + { + boost::algorithm::to_lower(matched_region); + region = matched_region; + } + } +} + PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfiguration) : per_request_configuration(clientConfiguration.perRequestConfiguration) diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index eefc85fae70..25055754519 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -19,6 +19,8 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration const RemoteHostFilter & remote_host_filter; PocoHTTPClientConfiguration(const Aws::Client::ClientConfiguration & cfg, const RemoteHostFilter & remote_host_filter_); + + void updateSchemeAndRegion(); }; class PocoHTTPClient : public Aws::Http::HttpClient diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index db7aaf1549b..1304b6b5054 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -13,7 +13,6 @@ # include # include # include -# include # include # include # include @@ -186,20 +185,7 @@ namespace S3 PocoHTTPClientConfiguration client_configuration(cfg, remote_host_filter); - if (!client_configuration.endpointOverride.empty()) - { - static const RE2 region_pattern(R"(^s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); - Poco::URI uri(client_configuration.endpointOverride); - if (uri.getScheme() == "http") - client_configuration.scheme = Aws::Http::Scheme::HTTP; - - String region; - if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, ®ion)) - { - boost::algorithm::to_lower(region); - client_configuration.region = region; - } - } + client_configuration.updateSchemeAndRegion(); return std::make_shared( credentials, // Aws credentials. @@ -217,16 +203,18 @@ namespace S3 HeaderCollection headers, const RemoteHostFilter & remote_host_filter) { - PocoHTTPClientConfiguration cfg({}, remote_host_filter); + PocoHTTPClientConfiguration client_configuration({}, remote_host_filter); if (!endpoint.empty()) - cfg.endpointOverride = endpoint; + client_configuration.endpointOverride = endpoint; + + client_configuration.updateSchemeAndRegion(); Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); return std::make_shared( - std::make_shared(cfg, std::move(credentials), std::move(headers)), - std::move(cfg), // Client configuration. - is_virtual_hosted_style || cfg.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified. + std::make_shared(client_configuration, std::move(credentials), std::move(headers)), + std::move(client_configuration), // Client configuration. + is_virtual_hosted_style || client_configuration.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified. ); } From 8320efc18975b4ef591f6f2a4a243908ab216816 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 11:24:12 +0300 Subject: [PATCH 108/441] Fix read from Buffer table with empty nested table with different types. --- src/Storages/StorageBuffer.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 4b038692b77..659df2026c8 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -221,17 +221,20 @@ Pipe StorageBuffer::read( columns_intersection, destination_metadata_snapshot, query_info, context, processed_stage, max_block_size, num_streams); - pipe_from_dst.addSimpleTransform([&](const Block & stream_header) + if (!pipe_from_dst.empty()) { - return std::make_shared(stream_header, header_after_adding_defaults, - metadata_snapshot->getColumns(), context); - }); + pipe_from_dst.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared(stream_header, header_after_adding_defaults, + metadata_snapshot->getColumns(), context); + }); - pipe_from_dst.addSimpleTransform([&](const Block & stream_header) - { - return std::make_shared( - stream_header, header, ConvertingTransform::MatchColumnsMode::Name); - }); + pipe_from_dst.addSimpleTransform([&](const Block & stream_header) + { + return std::make_shared( + stream_header, header, ConvertingTransform::MatchColumnsMode::Name); + }); + } } } From 765db4196f2edba7d6f10141c239d01a3c359f11 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 13:35:34 +0300 Subject: [PATCH 109/441] Add test. --- .../01514_empty_buffer_different_types.reference | 0 .../0_stateless/01514_empty_buffer_different_types.sql | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/01514_empty_buffer_different_types.reference create mode 100644 tests/queries/0_stateless/01514_empty_buffer_different_types.sql diff --git a/tests/queries/0_stateless/01514_empty_buffer_different_types.reference b/tests/queries/0_stateless/01514_empty_buffer_different_types.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_empty_buffer_different_types.sql b/tests/queries/0_stateless/01514_empty_buffer_different_types.sql new file mode 100644 index 00000000000..ce3c9edc39d --- /dev/null +++ b/tests/queries/0_stateless/01514_empty_buffer_different_types.sql @@ -0,0 +1,8 @@ +set send_logs_level = 'error'; + +DROP TABLE IF EXISTS merge_tree_table1; +CREATE TABLE merge_tree_table1 (`s` LowCardinality(String)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE buffer_table1 ( `s` String ) ENGINE = Buffer(currentDatabase(), 'merge_tree_table1', 16, 10, 60, 10, 1000, 1048576, 2097152); +SELECT * FROM buffer_table1; + +DROP TABLE IF EXISTS merge_tree_table1; From 563ad62e1c7e5f2ff528464d646e4e0e01a12005 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 6 Oct 2020 13:44:05 +0300 Subject: [PATCH 110/441] Update test. --- .../01514_empty_buffer_different_types.reference | 1 + .../0_stateless/01514_empty_buffer_different_types.sql | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01514_empty_buffer_different_types.reference b/tests/queries/0_stateless/01514_empty_buffer_different_types.reference index e69de29bb2d..78981922613 100644 --- a/tests/queries/0_stateless/01514_empty_buffer_different_types.reference +++ b/tests/queries/0_stateless/01514_empty_buffer_different_types.reference @@ -0,0 +1 @@ +a diff --git a/tests/queries/0_stateless/01514_empty_buffer_different_types.sql b/tests/queries/0_stateless/01514_empty_buffer_different_types.sql index ce3c9edc39d..3afadbcd33f 100644 --- a/tests/queries/0_stateless/01514_empty_buffer_different_types.sql +++ b/tests/queries/0_stateless/01514_empty_buffer_different_types.sql @@ -1,8 +1,12 @@ set send_logs_level = 'error'; DROP TABLE IF EXISTS merge_tree_table1; -CREATE TABLE merge_tree_table1 (`s` LowCardinality(String)) ENGINE = MergeTree ORDER BY tuple(); -CREATE TABLE buffer_table1 ( `s` String ) ENGINE = Buffer(currentDatabase(), 'merge_tree_table1', 16, 10, 60, 10, 1000, 1048576, 2097152); -SELECT * FROM buffer_table1; +CREATE TABLE merge_tree_table1 (`s` LowCardinality(String), x UInt32) ENGINE = MergeTree ORDER BY x settings index_granularity = 1; +CREATE TABLE buffer_table1 ( `s` String , x UInt32) ENGINE = Buffer(currentDatabase(), 'merge_tree_table1', 16, 10, 60, 10, 1000, 1048576, 2097152); +SELECT s FROM buffer_table1; + +insert into merge_tree_table1 values ('a', 1); +select s from buffer_table1 where x = 1; +select s from buffer_table1 where x = 2; DROP TABLE IF EXISTS merge_tree_table1; From 064be8db54630550bf0736a1de9eff1ad57eea4b Mon Sep 17 00:00:00 2001 From: Anna Date: Tue, 6 Oct 2020 16:16:28 +0300 Subject: [PATCH 111/441] Add desc for Default codec --- docs/en/sql-reference/statements/create/table.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index dbe1f282b5d..0290b7b32ee 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -121,7 +121,9 @@ Defines storage time for values. Can be specified only for MergeTree-family tabl ## Column Compression Codecs {#codecs} -By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. You can also define the compression method for each individual column in the `CREATE TABLE` query. +By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. + +You can also define the compression method for each individual column in the `CREATE TABLE` query. ``` sql CREATE TABLE codec_example @@ -136,7 +138,16 @@ ENGINE = ... ``` -If a codec is specified, the default codec doesn’t apply. Codecs can be combined in a pipeline, for example, `CODEC(Delta, ZSTD)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. +If the `Default` codec is specified explicitly, the default compression replaced with current default codec which may dependend on different settings (and properties of data) in runtime. In this case, the default compression is set in the CODEC definition: +`value UInt64 CODEC(Default)` - the same as lack of codec specification. + +Also you can remove current CODEC from the column and use default compression from config.xml: + +``` sql +ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); +``` + +Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. !!! warning "Warning" You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. From 950a07835f287fba38b4c1e018a0588157bcfe9f Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 19:23:08 +0400 Subject: [PATCH 112/441] Stylistic changes --- src/Access/IAccessStorage.cpp | 7 ++++--- src/Access/IAccessStorage.h | 2 +- src/Access/MultipleAccessStorage.cpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index b21527e48f1..8dd219e07d7 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -421,14 +421,15 @@ UUID IAccessStorage::login( const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, - bool log_and_mask_exceptions) const + bool replace_exception_with_cannot_authenticate) const { - try { + try + { return loginImpl(user_name, password, address, external_authenticators); } catch (...) { - if (!log_and_mask_exceptions) + if (!replace_exception_with_cannot_authenticate) throw; tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 93c97144cda..ecf6b260712 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -144,7 +144,7 @@ public: /// Finds an user, check its password and returns the ID of the user. /// Throws an exception if no such user or password is incorrect. - UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool log_and_mask_exceptions = true) const; + UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate = true) const; /// Returns the ID of an user who has logged in (maybe on another node). /// The function assumes that the password has been already checked somehow, so we can skip checking it now. diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 32aa8c50159..6f888f2f150 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -409,7 +409,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p { try { - auto id = storage->login(user_name, password, address, external_authenticators, false); + auto id = storage->login(user_name, password, address, external_authenticators, /* replace_exception_with_cannot_authenticate = */ false); std::lock_guard lock{mutex}; ids_cache.set(id, storage); return id; From e2f444ae8559f972e89d62409a84b98aea1b3986 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 19:37:35 +0400 Subject: [PATCH 113/441] Simplify loginImpl() and getIDOfLoggedUserImpl() --- src/Access/MultipleAccessStorage.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 6f888f2f150..86790a23933 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -30,6 +30,7 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_) MultipleAccessStorage::~MultipleAccessStorage() { + /// It's better to remove the storages in the reverse order because they could depend on each other somehow. const auto storages = getStoragesPtr(); for (const auto & storage : *storages | boost::adaptors::reversed) { @@ -414,9 +415,9 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p ids_cache.set(id, storage); return id; } - catch (...) + catch (const Exception & e) { - if (!storage->find(EntityType::USER, user_name)) + if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -441,9 +442,9 @@ UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) cons ids_cache.set(id, storage); return id; } - catch (...) + catch (const Exception & e) { - if (!storage->find(EntityType::USER, user_name)) + if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. From dfc13ca8e7d68a1cfb9fb8ccca494a44c7d82b19 Mon Sep 17 00:00:00 2001 From: Vasily Kozhukhovskiy Date: Tue, 6 Oct 2020 18:37:54 +0300 Subject: [PATCH 114/441] Enable parsing enum values by their ids for CSV, TSV and JSON input formats * for CSV and TSV input formats input_format_csv_enum_as_number, input_format_tsv_enum_as_number settings should be enabled in order to treat input value as enum id --- docs/en/operations/settings/settings.md | 8 +++ src/Core/Settings.h | 2 + src/DataTypes/DataTypeEnum.cpp | 67 ++++++++++++++----- src/Formats/FormatFactory.cpp | 2 + src/Formats/FormatSettings.h | 2 + ...ormat_csv_enum_as_number_setting.reference | 0 ...nput_format_csv_enum_as_number_setting.sql | 14 ++++ ...input_format_json_enum_as_number.reference | 0 ...01514_input_format_json_enum_as_number.sql | 10 +++ ...ormat_tsv_enum_as_number_setting.reference | 0 ...nput_format_tsv_enum_as_number_setting.sql | 14 ++++ 11 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.reference create mode 100644 tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql create mode 100644 tests/queries/0_stateless/01514_input_format_json_enum_as_number.reference create mode 100644 tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql create mode 100644 tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.reference create mode 100644 tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index decaf6b9029..b5fda4c5699 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -276,6 +276,10 @@ When enabled, replace empty input fields in TSV with default values. For complex Disabled by default. +## input\_format\_tsv\_enum\_as\_number {#settings-input_format_tsv_enum_as_number} + +For TSV input format switches to parsing enum values as enum ids. + ## input\_format\_null\_as\_default {#settings-input-format-null-as-default} Enables or disables using default values if input data contain `NULL`, but data type of the corresponding column in not `Nullable(T)` (for text input formats). @@ -1107,6 +1111,10 @@ The character interpreted as a delimiter in the CSV data. By default, the delimi For CSV input format enables or disables parsing of unquoted `NULL` as literal (synonym for `\N`). +## input\_format\_csv\_enum\_as\_number {#settings-input_format_csv_enum_as_number} + +For CSV input format switches to parsing enum values as enum ids. + ## output\_format\_csv\_crlf\_end\_of\_line {#settings-output-format-csv-crlf-end-of-line} Use DOS/Windows-style line separator (CRLF) in CSV instead of Unix style (LF). diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3ecb79c3fce..7c13cf2cf58 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -412,12 +412,14 @@ class IColumn; M(Bool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.", 0) \ M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \ M(Bool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N", 0) \ + M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices \\N", 0) \ M(Bool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).", 0) \ M(Bool, input_format_with_names_use_header, true, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ M(Bool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, CSV and TSV formats).", IMPORTANT) \ M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ + M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices \\N", 0) \ M(Bool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \ diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/DataTypeEnum.cpp index 9ad6a9cb690..86049e41e68 100644 --- a/src/DataTypes/DataTypeEnum.cpp +++ b/src/DataTypes/DataTypeEnum.cpp @@ -146,12 +146,22 @@ void DataTypeEnum::serializeTextEscaped(const IColumn & column, size_t row } template -void DataTypeEnum::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void DataTypeEnum::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - /// NOTE It would be nice to do without creating a temporary object - at least extract std::string out. - std::string field_name; - readEscapedString(field_name, istr); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (settings.tsv.input_format_enum_as_number) + { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + /// NOTE It would be nice to do without creating a temporary object - at least extract std::string out. + std::string field_name; + readEscapedString(field_name, istr); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template @@ -169,11 +179,20 @@ void DataTypeEnum::deserializeTextQuoted(IColumn & column, ReadBuffer & is } template -void DataTypeEnum::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void DataTypeEnum::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - std::string field_name; - readString(field_name, istr); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (settings.tsv.input_format_enum_as_number) { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + std::string field_name; + readString(field_name, istr); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template @@ -191,9 +210,18 @@ void DataTypeEnum::serializeTextXML(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - std::string field_name; - readJSONString(field_name, istr); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (*istr.position() != '"') { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + std::string field_name; + readJSONString(field_name, istr); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template @@ -205,9 +233,18 @@ void DataTypeEnum::serializeTextCSV(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - std::string field_name; - readCSVString(field_name, istr, settings.csv); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (settings.csv.input_format_enum_as_number) { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + std::string field_name; + readCSVString(field_name, istr, settings.csv); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 065b14f86b7..a8b074df2a0 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -48,6 +48,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null; format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields; + format_settings.csv.input_format_enum_as_number = settings.input_format_csv_enum_as_number; format_settings.null_as_default = settings.input_format_null_as_default; format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; @@ -62,6 +63,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con format_settings.template_settings.row_format = settings.format_template_row; format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; + format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; format_settings.schema.format_schema = settings.format_schema; format_settings.schema.format_schema_path = context.getFormatSchemaPath(); format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index a97bd9bf6c6..1ba9949a6b3 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -34,6 +34,7 @@ struct FormatSettings bool unquoted_null_literal_as_null = false; bool empty_as_default = false; bool crlf_end_of_line = false; + bool input_format_enum_as_number = false; }; CSV csv; @@ -81,6 +82,7 @@ struct FormatSettings bool empty_as_default = false; bool crlf_end_of_line = false; String null_representation = "\\N"; + bool input_format_enum_as_number = false; }; TSV tsv; diff --git a/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.reference b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql new file mode 100644 index 00000000000..f4f2b278ac3 --- /dev/null +++ b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS table_with_enum_column_for_csv_insert; + +CREATE TABLE table_with_enum_column_for_csv_insert ( + Id Int32, + Value Enum('ef' = 1, 'es' = 2) +) ENGINE=Memory(); + +SET input_format_csv_enum_as_number = 1; + +INSERT INTO table_with_enum_column_for_csv_insert FORMAT CSV 102,2 + +SET input_format_csv_enum_as_number = 0; + +DROP TABLE IF EXISTS table_with_enum_column_for_csv_insert; diff --git a/tests/queries/0_stateless/01514_input_format_json_enum_as_number.reference b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql new file mode 100644 index 00000000000..a07cf87e11d --- /dev/null +++ b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS table_with_enum_column_for_json_insert; + +CREATE TABLE table_with_enum_column_for_json_insert ( + Id Int32, + Value Enum('ef' = 1, 'es' = 2) +) ENGINE=Memory(); + +INSERT INTO table_with_enum_column_for_json_insert FORMAT JSONEachRow {"Id":102,"Value":2} + +DROP TABLE IF EXISTS table_with_enum_column_for_json_insert; diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.reference b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql new file mode 100644 index 00000000000..f514c0f4dc5 --- /dev/null +++ b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS table_with_enum_column_for_tsv_insert; + +CREATE TABLE table_with_enum_column_for_tsv_insert ( + Id Int32, + Value Enum('ef' = 1, 'es' = 2) +) ENGINE=Memory(); + +SET input_format_tsv_enum_as_number = 1; + +INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2 + +SET input_format_tsv_enum_as_number = 0; + +DROP TABLE IF EXISTS table_with_enum_column_for_tsv_insert; From fed6080273eb7d1e837dcf6ca8be8e5ed1d3a10c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 19:54:22 +0400 Subject: [PATCH 115/441] Implement custom getIDOfLoggedUserImpl() --- src/Access/LDAPAccessStorage.cpp | 30 ++++++++++++++++++++++++++++++ src/Access/LDAPAccessStorage.h | 1 + 2 files changed, 31 insertions(+) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 9b1c6a48a13..d1dbf10cfbc 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -271,6 +271,36 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw } } +UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const +{ + std::scoped_lock lock(mutex); + auto id = memory_storage.find(user_name); + if (id) + { + return *id; + } + else + { + // User does not exist, so we create one, and add it pretending that the authentication is successful. + auto user = std::make_shared(); + user->setName(user_name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + for (const auto& role_name : default_role_names) + { + std::optional role_id = access_control_manager->find(role_name); + if (!role_id) + throwDefaultRoleNotFound(role_name); + + roles_of_interest.insert(role_id.value()); + user->granted_roles.grant(role_id.value()); + } + + return memory_storage.insert(user); + } +} + void LDAPAccessStorage::throwDefaultRoleNotFound(const String & role_name) { throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 02c44a8d400..1e6e0713568 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -50,6 +50,7 @@ private: // IAccessStorage implementations. virtual bool hasSubscriptionImpl(const UUID & id) const override; virtual bool hasSubscriptionImpl(EntityType type) const override; virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; + virtual UUID getIDOfLoggedUserImpl(const String & user_name) const override; private: void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); From 54446eeec6e6f6c4449a45c6d78a015a0fb0f54c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 20:00:29 +0400 Subject: [PATCH 116/441] Use ErrorCodes::UNKNOWN_USER --- src/Access/MultipleAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 86790a23933..170819922ec 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -417,7 +417,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p } catch (const Exception & e) { - if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) + if (e.code() == ErrorCodes::UNKNOWN_USER) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -444,7 +444,7 @@ UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) cons } catch (const Exception & e) { - if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) + if (e.code() == ErrorCodes::UNKNOWN_USER) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. From 555f056a4c13c29d7f7a541bc135a1a7c98b59e8 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 20:32:06 +0400 Subject: [PATCH 117/441] Revert "user not found" detection in loginImpl() and getIDOfLoggedUserImpl() --- src/Access/MultipleAccessStorage.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 170819922ec..a8ce3f602ed 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -415,9 +415,9 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p ids_cache.set(id, storage); return id; } - catch (const Exception & e) + catch (...) { - if (e.code() == ErrorCodes::UNKNOWN_USER) + if (!storage->find(EntityType::USER, user_name)) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -442,9 +442,9 @@ UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) cons ids_cache.set(id, storage); return id; } - catch (const Exception & e) + catch (...) { - if (e.code() == ErrorCodes::UNKNOWN_USER) + if (!storage->find(EntityType::USER, user_name)) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. From f549ecf9d1f278eb4792f03cb7a8fee80bc1de39 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Tue, 6 Oct 2020 21:49:48 +0000 Subject: [PATCH 118/441] insert_quorum_parallel: status nodes in /quorum/parallel and allow merges --- .../MergeTree/ReplicatedMergeTreeBlockEntry.h | 69 -------- .../ReplicatedMergeTreeBlockOutputStream.cpp | 59 ++----- .../ReplicatedMergeTreeCleanupThread.cpp | 16 +- .../ReplicatedMergeTreePartHeader.cpp | 19 --- .../MergeTree/ReplicatedMergeTreePartHeader.h | 11 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 37 +---- .../MergeTree/ReplicatedMergeTreeQueue.h | 2 +- .../ReplicatedMergeTreeQuorumEntry.h | 28 +++- .../ReplicatedMergeTreeQuorumStatusEntry.h | 76 --------- .../ReplicatedMergeTreeRestartingThread.cpp | 35 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 154 +++++++++--------- src/Storages/StorageReplicatedMergeTree.h | 9 +- 12 files changed, 141 insertions(+), 374 deletions(-) delete mode 100644 src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h delete mode 100644 src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h deleted file mode 100644 index 7d561c7fc6f..00000000000 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockEntry.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -/** To implement the functionality of the "quorum write". - * Information about which replicas the inserted part of data appeared on, - * and on how many replicas it should be. - */ -struct ReplicatedMergeTreeBlockEntry -{ - String part_name; - std::optional quorum_status; - - ReplicatedMergeTreeBlockEntry() {} - ReplicatedMergeTreeBlockEntry(const String & str) - { - fromString(str); - } - - void writeText(WriteBuffer & out) const - { - out << part_name; - - if (quorum_status) - { - out << "\n"; - quorum_status->writeText(out); - } - } - - void readText(ReadBuffer & in) - { - in >> part_name; - - if (!in.eof()) - { - in >> "\n"; - quorum_status = ReplicatedMergeTreeQuorumStatusEntry(); - quorum_status->readText(in); - } - } - - String toString() const - { - WriteBufferFromOwnString out; - writeText(out); - return out.str(); - } - - void fromString(const String & str) - { - ReadBufferFromString in(str); - readText(in); - } -}; - -} diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 67ce355bb7f..69e49fccb7c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -246,12 +245,10 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( Int64 block_number = 0; String existing_part_name; - ReplicatedMergeTreeBlockEntry block_entry; if (block_number_lock) { is_already_existing_part = false; block_number = block_number_lock->getNumber(); - block_entry.part_name = part->name; /// Set part attributes according to part_number. Prepare an entry for log. @@ -287,9 +284,10 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( */ if (quorum) { - ReplicatedMergeTreeQuorumStatusEntry status_entry; - status_entry.required_number_of_replicas = quorum; - status_entry.replicas.insert(storage.replica_name); + ReplicatedMergeTreeQuorumEntry quorum_entry; + quorum_entry.part_name = part->name; + quorum_entry.required_number_of_replicas = quorum; + quorum_entry.replicas.insert(storage.replica_name); /** At this point, this node will contain information that the current replica received a part. * When other replicas will receive this part (in the usual way, processing the replication log), @@ -298,20 +296,14 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( * which indicates that the quorum has been reached. */ - if (!quorum_parallel) - { - ReplicatedMergeTreeQuorumEntry quorum_entry; - quorum_entry.part_name = part->name; - quorum_entry.status = status_entry; + if (quorum_parallel) + quorum_info.status_path = storage.zookeeper_path + "/quorum/parallel/" + part->name; - ops.emplace_back( - zkutil::makeCreateRequest( - quorum_info.status_path, - quorum_entry.toString(), - zkutil::CreateMode::Persistent)); - } - else - block_entry.quorum_status = status_entry; + ops.emplace_back( + zkutil::makeCreateRequest( + quorum_info.status_path, + quorum_entry.toString(), + zkutil::CreateMode::Persistent)); /// Make sure that during the insertion time, the replica was not reinitialized or disabled (when the server is finished). ops.emplace_back( @@ -337,9 +329,8 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( /// Note: race condition with DROP PARTITION operation is possible. User will get "No node" exception and it is Ok. existing_part_name = zookeeper->get(storage.zookeeper_path + "/blocks/" + block_id); - block_entry.fromString(existing_part_name); /// If it exists on our replica, ignore it. - if (storage.getActiveContainingPart(block_entry.part_name)) + if (storage.getActiveContainingPart(existing_part_name)) { LOG_INFO(log, "Block with ID {} already exists locally as part {}; ignoring it.", block_id, existing_part_name); part->is_duplicate = true; @@ -360,13 +351,12 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( /// Used only for exception messages. block_number = part->info.min_block; - /// Do not check for duplicate on commit to ZK. block_id_path.clear(); } /// Information about the part. - storage.getCommitPartOps(ops, part, block_id_path, block_entry); + storage.getCommitPartOps(ops, part, block_id_path); MergeTreeData::Transaction transaction(storage); /// If you can not add a part to ZK, we'll remove it back from the working set. bool renamed = false; @@ -486,8 +476,6 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( /// We are waiting for quorum to be satisfied. LOG_TRACE(log, "Waiting for quorum"); - String quorum_status_path = quorum_parallel ? storage.zookeeper_path + "/blocks/" + block_id : storage.zookeeper_path + "/quorum/status"; - try { while (true) @@ -495,26 +483,15 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( zkutil::EventPtr event = std::make_shared(); std::string value; - ReplicatedMergeTreeQuorumEntry quorum_entry; - ReplicatedMergeTreeBlockEntry block_entry; /// `get` instead of `exists` so that `watch` does not leak if the node is no longer there. - if (!zookeeper->tryGet(quorum_status_path, value, nullptr, event)) + if (!zookeeper->tryGet(quorum_info.status_path, value, nullptr, event)) break; + ReplicatedMergeTreeQuorumEntry quorum_entry(value); + /// If the node has time to disappear, and then appear again for the next insert. - if (quorum_parallel) - { - block_entry.fromString(value); - // quorum_status empty if quorum reached - if (block_entry.part_name != part->name || !block_entry.quorum_status) - break; - } - else - { - quorum_entry.fromString(value); - if (quorum_entry.part_name != part->name) - break; - } + if (quorum_entry.part_name != part->name) + break; if (!event->tryWait(quorum_timeout_ms)) throw Exception("Timeout while waiting for quorum", ErrorCodes::TIMEOUT_EXCEEDED); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 03d920b2033..11f23a5c110 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -340,25 +340,11 @@ void ReplicatedMergeTreeCleanupThread::clearOldBlocks() timed_blocks.begin(), timed_blocks.end(), block_threshold, NodeWithStat::greaterByTime); auto first_outdated_block = std::min(first_outdated_block_fixed_threshold, first_outdated_block_time_threshold); - zkutil::AsyncResponses try_get_futures; zkutil::AsyncResponses try_remove_futures; for (auto it = first_outdated_block; it != timed_blocks.end(); ++it) { String path = storage.zookeeper_path + "/blocks/" + it->node; - try_get_futures.emplace_back(path, zookeeper->asyncTryGet(path)); - } - - /// Don't delete blocks that are still reaching quorum - for (auto & pair : try_get_futures) - { - const String & path = pair.first; - auto response = pair.second.get(); - if (response.error == Coordination::Error::ZOK) - { - ReplicatedMergeTreeBlockEntry block(response.data); - if (!block.quorum_status) - try_remove_futures.emplace_back(path, zookeeper->asyncTryRemove(path)); - } + try_remove_futures.emplace_back(path, zookeeper->asyncTryRemove(path)); } for (auto & pair : try_remove_futures) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp index 3f9c86d7c49..88f4a3ec66f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.cpp @@ -34,28 +34,11 @@ ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromColumnsAndCheck return ReplicatedMergeTreePartHeader(getSipHash(columns.toString()), std::move(checksums)); } -ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromColumnsChecksumsBlockID( - const NamesAndTypesList & columns, - const MergeTreeDataPartChecksums & full_checksums, - const String & block_id_name) -{ - MinimalisticDataPartChecksums checksums; - checksums.computeTotalChecksums(full_checksums); - return ReplicatedMergeTreePartHeader(getSipHash(columns.toString()), std::move(checksums), block_id_name); -} - void ReplicatedMergeTreePartHeader::read(ReadBuffer & in) { in >> "part header format version: 1\n"; in.readStrict(columns_hash.data(), columns_hash.size()); checksums.deserializeWithoutHeader(in); - - if (!in.eof()) - { - String new_block_id; - in >> "\nblock_id: " >> new_block_id; - block_id = new_block_id; - } } ReplicatedMergeTreePartHeader ReplicatedMergeTreePartHeader::fromString(const String & str) @@ -71,8 +54,6 @@ void ReplicatedMergeTreePartHeader::write(WriteBuffer & out) const writeString("part header format version: 1\n", out); out.write(columns_hash.data(), columns_hash.size()); checksums.serializeWithoutHeader(out); - if (block_id) - out << "\nblock_id: " << block_id.value(); } String ReplicatedMergeTreePartHeader::toString() const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h index a5cca569b56..7bc2b72d2d5 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartHeader.h @@ -6,7 +6,6 @@ #include #include #include -#include namespace DB @@ -29,9 +28,6 @@ public: static ReplicatedMergeTreePartHeader fromColumnsAndChecksums( const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums); - static ReplicatedMergeTreePartHeader fromColumnsChecksumsBlockID( - const NamesAndTypesList & columns, const MergeTreeDataPartChecksums & full_checksums, const String & block_id_name); - void read(ReadBuffer & in); static ReplicatedMergeTreePartHeader fromString(const String & str); @@ -40,18 +36,15 @@ public: const std::array & getColumnsHash() const { return columns_hash; } const MinimalisticDataPartChecksums & getChecksums() const { return checksums; } - const std::optional & getBlockID() const { return block_id; } private: - ReplicatedMergeTreePartHeader(std::array columns_hash_, MinimalisticDataPartChecksums checksums_, - std::optional block_id_ = std::nullopt) - : columns_hash(std::move(columns_hash_)), checksums(std::move(checksums_)), block_id(std::move(block_id_)) + ReplicatedMergeTreePartHeader(std::array columns_hash_, MinimalisticDataPartChecksums checksums_) + : columns_hash(std::move(columns_hash_)), checksums(std::move(checksums_)) { } std::array columns_hash; MinimalisticDataPartChecksums checksums; - std::optional block_id; }; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index c88172f0701..ef7ebead966 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include @@ -1698,6 +1697,9 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( prev_virtual_parts = queue.virtual_parts; } + /// Load current quorum status. + auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); + /// Load current inserts std::unordered_set lock_holder_paths; for (const String & entry : zookeeper->getChildren(queue.zookeeper_path + "/temp")) @@ -1749,38 +1751,15 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( merges_version = queue_.pullLogsToQueue(zookeeper); - /// Load current quorum status. - auto quorum_status_future = zookeeper->asyncTryGet(queue.zookeeper_path + "/quorum/status"); Coordination::GetResponse quorum_status_response = quorum_status_future.get(); if (quorum_status_response.error == Coordination::Error::ZOK) { ReplicatedMergeTreeQuorumEntry quorum_status; quorum_status.fromString(quorum_status_response.data); - inprogress_quorum_parts.insert(quorum_status.part_name); + inprogress_quorum_part = quorum_status.part_name; } else - inprogress_quorum_parts.clear(); - - Strings partitions = zookeeper->getChildren(queue.replica_path + "/parts"); - for (const String & partition : partitions) - { - auto part_str = zookeeper->get(queue.replica_path + "/parts/" + partition); - if (part_str.empty()) - { - /// use_minimalistic_part_header_in_zookeeper - continue; - } - auto header = ReplicatedMergeTreePartHeader::fromString(part_str); - if (header.getBlockID()) - { - ReplicatedMergeTreeBlockEntry block(zookeeper->get(queue.zookeeper_path + "/blocks/" + *header.getBlockID())); - if (partition != block.part_name) - throw Exception("partition " + partition + " contain block_id " + *header.getBlockID() + - " and block_id contain another partition " + block.part_name, ErrorCodes::LOGICAL_ERROR); - if (block.quorum_status) - inprogress_quorum_parts.insert(block.part_name); - } - } + inprogress_quorum_part.clear(); } bool ReplicatedMergeTreeMergePredicate::operator()( @@ -1842,7 +1821,7 @@ bool ReplicatedMergeTreeMergePredicate::canMergeTwoParts( for (const MergeTreeData::DataPartPtr & part : {left, right}) { - if (inprogress_quorum_parts.count(part->name)) + if (part->name == inprogress_quorum_part) { if (out_reason) *out_reason = "Quorum insert for part " + part->name + " is currently in progress"; @@ -1937,7 +1916,7 @@ bool ReplicatedMergeTreeMergePredicate::canMergeSinglePart( const MergeTreeData::DataPartPtr & part, String * out_reason) const { - if (inprogress_quorum_parts.count(part->name)) + if (part->name == inprogress_quorum_part) { if (out_reason) *out_reason = "Quorum insert for part " + part->name + " is currently in progress"; @@ -1978,7 +1957,7 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir /// We cannot mutate part if it's being inserted with quorum and it's not /// already reached. - if (inprogress_quorum_parts.count(part->name)) + if (part->name == inprogress_quorum_part) return {}; std::lock_guard lock(queue.state_mutex); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index fb2733db667..88a61f50225 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -475,7 +475,7 @@ private: std::unordered_map> committing_blocks; /// Quorum state taken at some later time than prev_virtual_parts. - std::unordered_set inprogress_quorum_parts; + String inprogress_quorum_part; int32_t merges_version = -1; }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h index 93e117d82af..ee12cabb5aa 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h @@ -8,7 +8,6 @@ #include #include #include -#include namespace DB @@ -21,7 +20,8 @@ namespace DB struct ReplicatedMergeTreeQuorumEntry { String part_name; - ReplicatedMergeTreeQuorumStatusEntry status; + size_t required_number_of_replicas{}; + std::set replicas; ReplicatedMergeTreeQuorumEntry() {} ReplicatedMergeTreeQuorumEntry(const String & str) @@ -32,15 +32,31 @@ struct ReplicatedMergeTreeQuorumEntry void writeText(WriteBuffer & out) const { out << "version: 1\n" - << "part_name: " << part_name << "\n"; - status.writeText(out); + << "part_name: " << part_name << "\n" + << "required_number_of_replicas: " << required_number_of_replicas << "\n" + << "actual_number_of_replicas: " << replicas.size() << "\n" + << "replicas:\n"; + + for (const auto & replica : replicas) + out << escape << replica << "\n"; } void readText(ReadBuffer & in) { + size_t actual_number_of_replicas = 0; + in >> "version: 1\n" - >> "part_name: " >> part_name >> "\n"; - status.readText(in); + >> "part_name: " >> part_name >> "\n" + >> "required_number_of_replicas: " >> required_number_of_replicas >> "\n" + >> "actual_number_of_replicas: " >> actual_number_of_replicas >> "\n" + >> "replicas:\n"; + + for (size_t i = 0; i < actual_number_of_replicas; ++i) + { + String replica; + in >> escape >> replica >> "\n"; + replicas.insert(replica); + } } String toString() const diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h deleted file mode 100644 index bbf74808c2e..00000000000 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQuorumStatusEntry.h +++ /dev/null @@ -1,76 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -/** To implement the functionality of the "quorum write". - * Information about which replicas the inserted part of data appeared on, - * and on how many replicas it should be. - */ -struct ReplicatedMergeTreeQuorumStatusEntry -{ - size_t required_number_of_replicas{}; - std::set replicas; - - ReplicatedMergeTreeQuorumStatusEntry() {} - ReplicatedMergeTreeQuorumStatusEntry(const String & str) - { - fromString(str); - } - - void writeText(WriteBuffer & out) const - { - out << "required_number_of_replicas: " << required_number_of_replicas << "\n" - << "actual_number_of_replicas: " << replicas.size() << "\n" - << "replicas:\n"; - - for (const auto & replica : replicas) - out << escape << replica << "\n"; - } - - void readText(ReadBuffer & in) - { - size_t actual_number_of_replicas = 0; - - in >> "required_number_of_replicas: " >> required_number_of_replicas >> "\n" - >> "actual_number_of_replicas: " >> actual_number_of_replicas >> "\n" - >> "replicas:\n"; - - for (size_t i = 0; i < actual_number_of_replicas; ++i) - { - String replica; - in >> escape >> replica >> "\n"; - replicas.insert(replica); - } - } - - String toString() const - { - WriteBufferFromOwnString out; - writeText(out); - return out.str(); - } - - void fromString(const String & str) - { - ReadBufferFromString in(str); - readText(in); - } - - bool isQuorumReached() - { - return required_number_of_replicas && required_number_of_replicas <= replicas.size(); - } -}; - -} diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 908da215d42..e05ad01c908 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -228,10 +227,9 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() String quorum_str; if (zookeeper->tryGet(storage.zookeeper_path + "/quorum/status", quorum_str)) { - ReplicatedMergeTreeQuorumEntry quorum_entry; - quorum_entry.status.fromString(quorum_str); + ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); - if (!quorum_entry.status.replicas.count(storage.replica_name) + if (!quorum_entry.replicas.count(storage.replica_name) && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", quorum_entry.part_name); @@ -240,29 +238,20 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() } Strings partitions; - if (zookeeper->tryGetChildren(storage.replica_path + "/parts/", partitions) == Coordination::Error::ZOK) + String parallel_quorum_parts_path = storage.zookeeper_path + "/quorum/parallel"; + if (zookeeper->tryGetChildren(parallel_quorum_parts_path, partitions) == Coordination::Error::ZOK) { for (auto & partition : partitions) { - String part_str, block_str; - if (!zookeeper->tryGet(storage.replica_path + "/parts/" + partition, part_str)) - continue; - - auto header = ReplicatedMergeTreePartHeader::fromString(part_str); - if (!header.getBlockID() || !zookeeper->tryGet(storage.zookeeper_path + "/blocks/" + *header.getBlockID(), block_str)) - continue; - - ReplicatedMergeTreeBlockEntry block_entry(block_str); - if (block_entry.part_name != partition) + if (zookeeper->tryGet(parallel_quorum_parts_path + "/" + partition, quorum_str)) { - LOG_WARNING(log, "partition {} contain block_id {} and block_id contain another partition {}", partition, *header.getBlockID(), block_entry.part_name); - continue; - } - - if (block_entry.quorum_status && !block_entry.quorum_status->replicas.count(storage.replica_name)) - { - LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", block_entry.part_name); - storage.updateQuorum(block_entry.part_name, header.getBlockID()); + ReplicatedMergeTreeQuorumEntry quorum_entry(quorum_str); + if (!quorum_entry.replicas.count(storage.replica_name) + && zookeeper->exists(storage.replica_path + "/parts/" + partition)) + { + LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", partition); + storage.updateQuorum(partition); + } } } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 10b52ecc517..ed11dcb35c5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -474,6 +473,7 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() /// Working with quorum. zookeeper->createIfNotExists(zookeeper_path + "/quorum", String()); + zookeeper->createIfNotExists(zookeeper_path + "/quorum/parallel", String()); zookeeper->createIfNotExists(zookeeper_path + "/quorum/last_part", String()); zookeeper->createIfNotExists(zookeeper_path + "/quorum/failed_parts", String()); @@ -1701,28 +1701,31 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (replica.empty()) { - Coordination::Stat quorum_stat, block_stat; - String quorum_path = zookeeper_path + "/quorum/status"; - String block_path = entry.block_id.empty() ? "" : zookeeper_path + "/blocks/" + entry.block_id; - String quorum_str, parallel_quorum_str; + Coordination::Stat quorum_stat; + const String quorum_unparallel_path = zookeeper_path + "/quorum/status"; + const String quorum_parallel_path = zookeeper_path + "/quorum/parallel/" + entry.new_part_name; + String quorum_str, quorum_path; ReplicatedMergeTreeQuorumEntry quorum_entry; - ReplicatedMergeTreeBlockEntry block_entry; - if (zookeeper->tryGet(quorum_path, quorum_str, &quorum_stat)) - quorum_entry.fromString(quorum_str); - if (!block_path.empty() && zookeeper->tryGet(block_path, parallel_quorum_str, &block_stat)) - block_entry.fromString(parallel_quorum_str); - bool is_parallel_quorum = (block_entry.part_name == entry.new_part_name); + if (zookeeper->tryGet(quorum_unparallel_path, quorum_str, &quorum_stat)) + quorum_path = quorum_unparallel_path; + else { + quorum_str = zookeeper->get(quorum_parallel_path, &quorum_stat); + quorum_path = quorum_parallel_path; + } - if (quorum_entry.part_name == entry.new_part_name || is_parallel_quorum) + quorum_entry.fromString(quorum_str); + + if (quorum_entry.part_name == entry.new_part_name) { + ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); auto part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); + if (part_info.min_block != part_info.max_block) throw Exception("Logical error: log entry with quorum for part covering more than one block number", ErrorCodes::LOGICAL_ERROR); - if (!is_parallel_quorum) - ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); + ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); ops.emplace_back(zkutil::makeCreateRequest( zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name, @@ -1730,8 +1733,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) zkutil::CreateMode::Persistent)); /// Deleting from `blocks`. - if (!block_path.empty() && zookeeper->exists(block_path)) - ops.emplace_back(zkutil::makeRemoveRequest(block_path, -1)); + if (!entry.block_id.empty() && zookeeper->exists(zookeeper_path + "/blocks/" + entry.block_id)) + ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_path + "/blocks/" + entry.block_id, -1)); Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); @@ -1769,7 +1772,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (!entry.block_id.empty()) block_id = entry.block_id; String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; - if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum, nullptr, block_id)) + if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum)) return false; } catch (Exception & e) @@ -3040,74 +3043,70 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( /** If a quorum is tracked for a part, update information about it in ZK. */ -void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const std::optional & block_id) +void StorageReplicatedMergeTree::updateQuorum(const String & part_name) { auto zookeeper = getZooKeeper(); /// Information on which replicas a part has been added, if the quorum has not yet been reached. - const String quorum_status_path = zookeeper_path + "/quorum/status"; + const String quorum_unparallel_status_path = zookeeper_path + "/quorum/status"; /// The name of the previous part for which the quorum was reached. const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; - const String quorum_parallel_status_path = block_id ? zookeeper_path + "/blocks/" + *block_id : ""; + const String quorum_parallel_status_path = zookeeper_path + "/quorum/parallel/" + part_name; /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. String value; Coordination::Stat stat; while (true) { - bool is_parallel = false; - ReplicatedMergeTreeBlockEntry block_entry; + bool quorum_parallel(false); ReplicatedMergeTreeQuorumEntry quorum_entry; - if (zookeeper->tryGet(quorum_status_path, value, &stat)) - { - quorum_entry.fromString(value); - quorum_entry.status.replicas.insert(replica_name); - } - else if (block_id && zookeeper->tryGet(quorum_parallel_status_path, value, &stat)) - { - block_entry.fromString(value); - // The quorum has already been achieved - if (!block_entry.quorum_status) - break; - - is_parallel = true; - block_entry.quorum_status->replicas.insert(replica_name); - if (block_entry.quorum_status->isQuorumReached()) - block_entry.quorum_status.reset(); - } + if (zookeeper->tryGet(quorum_unparallel_status_path, value, &stat)) + {} + else if (zookeeper->tryGet(quorum_parallel_status_path, value, &stat)) + quorum_parallel = true; else break; - if (quorum_entry.part_name != part_name && block_entry.part_name != part_name) + const String quorum_status_path = quorum_parallel ? quorum_parallel_status_path : quorum_unparallel_status_path; + quorum_entry.fromString(value); + quorum_entry.replicas.insert(replica_name); + + if (quorum_entry.part_name != part_name) { /// The quorum has already been achieved. Moreover, another INSERT with a quorum has already started. break; } - if (quorum_entry.status.isQuorumReached()) + if (quorum_entry.replicas.size() >= quorum_entry.required_number_of_replicas) { /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. Coordination::Requests ops; Coordination::Responses responses; - Coordination::Stat added_parts_stat; - String old_added_parts = zookeeper->get(quorum_last_part_path, &added_parts_stat); + if (!quorum_parallel) + { + Coordination::Stat added_parts_stat; + String old_added_parts = zookeeper->get(quorum_last_part_path, &added_parts_stat); - ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version); + ReplicatedMergeTreeQuorumAddedParts parts_with_quorum(format_version); - if (!old_added_parts.empty()) - parts_with_quorum.fromString(old_added_parts); + if (!old_added_parts.empty()) + parts_with_quorum.fromString(old_added_parts); - auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); - /// We store one last part which reached quorum for each partition. - parts_with_quorum.added_parts[part_info.partition_id] = part_name; + auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); + /// We store one last part which reached quorum for each partition. + parts_with_quorum.added_parts[part_info.partition_id] = part_name; - String new_added_parts = parts_with_quorum.toString(); + String new_added_parts = parts_with_quorum.toString(); + + ops.emplace_back(zkutil::makeRemoveRequest(quorum_status_path, stat.version)); + ops.emplace_back(zkutil::makeSetRequest(quorum_last_part_path, new_added_parts, added_parts_stat.version)); + } + else + ops.emplace_back(zkutil::makeRemoveRequest(quorum_status_path, stat.version)); - ops.emplace_back(zkutil::makeRemoveRequest(quorum_status_path, stat.version)); - ops.emplace_back(zkutil::makeSetRequest(quorum_last_part_path, new_added_parts, added_parts_stat.version)); auto code = zookeeper->tryMulti(ops, responses); if (code == Coordination::Error::ZOK) @@ -3129,17 +3128,8 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, const st } else { - Coordination::Error code; - if (!is_parallel) - { - /// We update the node, registering there one more replica. - code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); - } - else - { - /// Update parallel quorum. It also might be reached here. - code = zookeeper->trySet(quorum_parallel_status_path, block_entry.toString(), stat.version); - } + /// We update the node, registering there one more replica. + auto code = zookeeper->trySet(quorum_status_path, quorum_entry.toString(), stat.version); if (code == Coordination::Error::ZOK) { @@ -3215,7 +3205,7 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, - const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_, const std::optional & block_id) + const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_) { auto zookeeper = zookeeper_ ? zookeeper_ : getZooKeeper(); const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -3345,7 +3335,20 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. */ if (quorum) - updateQuorum(part_name, block_id); + updateQuorum(part_name); + + /// alexelex: i'm not sure in witch order should that and merge_selecting_task->schedule() be + /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before + if (part_info.max_block != part_info.min_block) + { + Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); + for (const String & quorum_part : quorum_parts) + { + auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); + if (part_info.contains(quorum_part_info)) + updateQuorum(quorum_part); + } + } merge_selecting_task->schedule(); @@ -5283,11 +5286,10 @@ void StorageReplicatedMergeTree::clearBlocksInPartition( if (result.error == Coordination::Error::ZNONODE) continue; - ReplicatedMergeTreeBlockEntry block(result.data); - ReadBufferFromString buf(block.part_name); + ReadBufferFromString buf(result.data); Int64 block_num = 0; bool parsed = tryReadIntText(block_num, buf) && buf.eof(); - if ((!parsed || (min_block_num <= block_num && block_num <= max_block_num)) && !block.quorum_status) + if (!parsed || (min_block_num <= block_num && block_num <= max_block_num)) to_delete_futures.emplace_back(path, zookeeper.asyncTryRemove(path)); } @@ -5703,39 +5705,31 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta void StorageReplicatedMergeTree::getCommitPartOps( Coordination::Requests & ops, MutableDataPartPtr & part, - const String & block_id_path, - ReplicatedMergeTreeBlockEntry block_entry) const + const String & block_id_path) const { + const String & part_name = part->name; const auto storage_settings_ptr = getSettings(); if (!block_id_path.empty()) { /// Make final duplicate check and commit block_id - block_entry.part_name = part->name; ops.emplace_back( zkutil::makeCreateRequest( block_id_path, - block_entry.toString(), /// We will be able to know original part number for duplicate blocks, if we want. + part_name, /// We will be able to know original part number for duplicate blocks, if we want. zkutil::CreateMode::Persistent)); } /// Information about the part, in the replica - String part_str, block_id = !block_id_path.empty() && block_entry.quorum_status ? block_id_path.substr(block_id_path.find_last_of('/') + 1) : ""; if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { - if (!block_id.empty()) - part_str = ReplicatedMergeTreePartHeader::fromColumnsChecksumsBlockID(part->getColumns(), part->checksums, block_id).toString(); - else - part_str = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(); - ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, - part_str, + ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), zkutil::CreateMode::Persistent)); } else { - /// alexelex: if (!block_id.empty()) throw or skip it? ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, "", diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 8014ed600ea..2bc9265331d 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -351,8 +350,7 @@ private: bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; - void getCommitPartOps(Coordination::Requests & ops, MutableDataPartPtr & part, const String & block_id_path = "", - ReplicatedMergeTreeBlockEntry block_entry = ReplicatedMergeTreeBlockEntry()) const; + void getCommitPartOps(Coordination::Requests & ops, MutableDataPartPtr & part, const String & block_id_path = "") const; /// Adds actions to `ops` that remove a part from ZooKeeper. /// Set has_children to true for "old-style" parts (those with /columns and /checksums child znodes). @@ -486,8 +484,7 @@ private: const String & replica_path, bool to_detached, size_t quorum, - zkutil::ZooKeeper::Ptr zookeeper_ = nullptr, - const std::optional & block_id = std::nullopt); + zkutil::ZooKeeper::Ptr zookeeper_ = nullptr); /// Required only to avoid races between executeLogEntry and fetchPartition std::unordered_set currently_fetching_parts; @@ -495,7 +492,7 @@ private: /// With the quorum being tracked, add a replica to the quorum for the part. - void updateQuorum(const String & part_name, const std::optional & block_id = std::nullopt); + void updateQuorum(const String & part_name); /// Deletes info from quorum/last_part node for particular partition_id. void cleanLastPartNode(const String & partition_id); From 644091459f5b2ca909e8e58c11802f076a5b58c1 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Tue, 6 Oct 2020 22:01:30 +0000 Subject: [PATCH 119/441] style fixes and remove small changes --- .../ReplicatedMergeTreeRestartingThread.cpp | 1 - src/Storages/StorageReplicatedMergeTree.cpp | 14 +++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index e05ad01c908..9d5aa84b3ab 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ed11dcb35c5..8c89053e2de 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1180,7 +1180,6 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } else { - /// alexelex: if (local_part_header.getBlockID()) throw or skip it? ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( @@ -1718,15 +1717,13 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (quorum_entry.part_name == entry.new_part_name) { - ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); + ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); auto part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); if (part_info.min_block != part_info.max_block) throw Exception("Logical error: log entry with quorum for part covering more than one block number", ErrorCodes::LOGICAL_ERROR); - ops.emplace_back(zkutil::makeRemoveRequest(quorum_path, quorum_stat.version)); - ops.emplace_back(zkutil::makeCreateRequest( zookeeper_path + "/quorum/failed_parts/" + entry.new_part_name, "", @@ -1768,9 +1765,6 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) try { - std::optional block_id; - if (!entry.block_id.empty()) - block_id = entry.block_id; String part_name = entry.actual_new_part_name.empty() ? entry.new_part_name : entry.actual_new_part_name; if (!fetchPart(part_name, metadata_snapshot, zookeeper_path + "/replicas/" + replica, false, entry.quorum)) return false; @@ -3053,9 +3047,10 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; const String quorum_parallel_status_path = zookeeper_path + "/quorum/parallel/" + part_name; - /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. String value; Coordination::Stat stat; + + /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. while (true) { bool quorum_parallel(false); @@ -3070,7 +3065,6 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) const String quorum_status_path = quorum_parallel ? quorum_parallel_status_path : quorum_unparallel_status_path; quorum_entry.fromString(value); - quorum_entry.replicas.insert(replica_name); if (quorum_entry.part_name != part_name) { @@ -3078,6 +3072,8 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) break; } + quorum_entry.replicas.insert(replica_name); + if (quorum_entry.replicas.size() >= quorum_entry.required_number_of_replicas) { /// The quorum is reached. Delete the node, and update information about the last part that was successfully written with quorum. From 9077c92862cf5c81f28b3531bfc350ac1d48c53c Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Tue, 6 Oct 2020 22:36:42 +0000 Subject: [PATCH 120/441] style check fix --- src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8c89053e2de..55213bf5f40 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1708,7 +1708,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (zookeeper->tryGet(quorum_unparallel_path, quorum_str, &quorum_stat)) quorum_path = quorum_unparallel_path; - else { + else + { quorum_str = zookeeper->get(quorum_parallel_path, &quorum_stat); quorum_path = quorum_parallel_path; } From b8aa440912787b58be368ff51c7d7f6e67295b09 Mon Sep 17 00:00:00 2001 From: Vasily Kozhukhovskiy Date: Wed, 7 Oct 2020 12:24:54 +0300 Subject: [PATCH 121/441] style fixes --- src/DataTypes/DataTypeEnum.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/DataTypeEnum.cpp index 86049e41e68..f445b342e76 100644 --- a/src/DataTypes/DataTypeEnum.cpp +++ b/src/DataTypes/DataTypeEnum.cpp @@ -181,7 +181,8 @@ void DataTypeEnum::deserializeTextQuoted(IColumn & column, ReadBuffer & is template void DataTypeEnum::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - if (settings.tsv.input_format_enum_as_number) { + if (settings.tsv.input_format_enum_as_number) + { FieldType x; readText(x, istr); static_cast(getNameForValue(x)); @@ -210,7 +211,8 @@ void DataTypeEnum::serializeTextXML(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - if (*istr.position() != '"') { + if (*istr.position() != '"') + { FieldType x; readText(x, istr); static_cast(getNameForValue(x)); @@ -233,7 +235,8 @@ void DataTypeEnum::serializeTextCSV(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - if (settings.csv.input_format_enum_as_number) { + if (settings.csv.input_format_enum_as_number) + { FieldType x; readText(x, istr); static_cast(getNameForValue(x)); From 04c71314897056e18fb781b27b7f429a11e3bc1d Mon Sep 17 00:00:00 2001 From: Vasily Kozhukhovskiy Date: Wed, 7 Oct 2020 12:25:22 +0300 Subject: [PATCH 122/441] add test for inserting enum values by ids for TabSeparatedRaw format (input_format_tsv_enum_as_number setting) --- .../01514_input_format_tsv_enum_as_number_setting.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql index f514c0f4dc5..c1f0ba9167e 100644 --- a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql +++ b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql @@ -8,6 +8,7 @@ CREATE TABLE table_with_enum_column_for_tsv_insert ( SET input_format_tsv_enum_as_number = 1; INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2 +INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TabSeparatedRaw 103 1 SET input_format_tsv_enum_as_number = 0; From a43cac2c1add7d750aa8827c44d03fab62b59533 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Wed, 7 Oct 2020 11:28:48 +0000 Subject: [PATCH 123/441] changing the condition for updateQuorum, specialized it and reduced the number of calls to the zk --- .../ReplicatedMergeTreeBlockOutputStream.cpp | 13 ++++--- .../ReplicatedMergeTreeRestartingThread.cpp | 4 +-- src/Storages/StorageReplicatedMergeTree.cpp | 36 ++++++++----------- src/Storages/StorageReplicatedMergeTree.h | 2 +- 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 69e49fccb7c..30182697b21 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -77,7 +77,6 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe { quorum_info.status_path = storage.zookeeper_path + "/quorum/status"; - std::future quorum_status_future = zookeeper->asyncTryGet(quorum_info.status_path); std::future is_active_future = zookeeper->asyncTryGet(storage.replica_path + "/is_active"); std::future host_future = zookeeper->asyncTryGet(storage.replica_path + "/host"); @@ -99,9 +98,9 @@ void ReplicatedMergeTreeBlockOutputStream::checkQuorumPrecondition(zkutil::ZooKe * If the quorum is reached, then the node is deleted. */ - auto quorum_status = quorum_status_future.get(); - if (quorum_status.error != Coordination::Error::ZNONODE && !quorum_parallel) - throw Exception("Quorum for previous write has not been satisfied yet. Status: " + quorum_status.data, + String quorum_status; + if (!quorum_parallel && zookeeper->tryGet(quorum_info.status_path, quorum_status)) + throw Exception("Quorum for previous write has not been satisfied yet. Status: " + quorum_status, ErrorCodes::UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE); /// Both checks are implicitly made also later (otherwise there would be a race condition). @@ -470,7 +469,11 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( if (is_already_existing_part) { /// We get duplicate part without fetch - storage.updateQuorum(part->name); + /// Check if this quorum insert is parallel or not + if (zookeeper->exists(storage.zookeeper_path + "/quorum/status")) + storage.updateQuorum(part->name, false); + else if (zookeeper->exists(storage.zookeeper_path + "/quorum/parallel/" + part->name)) + storage.updateQuorum(part->name, true); } /// We are waiting for quorum to be satisfied. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 9d5aa84b3ab..29e8f55ecd0 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -232,7 +232,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() && zookeeper->exists(storage.replica_path + "/parts/" + quorum_entry.part_name)) { LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", quorum_entry.part_name); - storage.updateQuorum(quorum_entry.part_name); + storage.updateQuorum(quorum_entry.part_name, false); } } @@ -249,7 +249,7 @@ void ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart() && zookeeper->exists(storage.replica_path + "/parts/" + partition)) { LOG_WARNING(log, "We have part {} but we is not in quorum. Updating quorum. This shouldn't happen often.", partition); - storage.updateQuorum(partition); + storage.updateQuorum(partition, true); } } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 55213bf5f40..75cdfc5607d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3038,35 +3038,22 @@ String StorageReplicatedMergeTree::findReplicaHavingCoveringPart( /** If a quorum is tracked for a part, update information about it in ZK. */ -void StorageReplicatedMergeTree::updateQuorum(const String & part_name) +void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_parallel) { auto zookeeper = getZooKeeper(); /// Information on which replicas a part has been added, if the quorum has not yet been reached. - const String quorum_unparallel_status_path = zookeeper_path + "/quorum/status"; + const String quorum_status_path = is_parallel ? zookeeper_path + "/quorum/parallel/" + part_name : zookeeper_path + "/quorum/status"; /// The name of the previous part for which the quorum was reached. const String quorum_last_part_path = zookeeper_path + "/quorum/last_part"; - const String quorum_parallel_status_path = zookeeper_path + "/quorum/parallel/" + part_name; String value; Coordination::Stat stat; /// If there is no node, then all quorum INSERTs have already reached the quorum, and nothing is needed. - while (true) + while (zookeeper->tryGet(quorum_status_path, value, &stat)) { - bool quorum_parallel(false); - ReplicatedMergeTreeQuorumEntry quorum_entry; - - if (zookeeper->tryGet(quorum_unparallel_status_path, value, &stat)) - {} - else if (zookeeper->tryGet(quorum_parallel_status_path, value, &stat)) - quorum_parallel = true; - else - break; - - const String quorum_status_path = quorum_parallel ? quorum_parallel_status_path : quorum_unparallel_status_path; - quorum_entry.fromString(value); - + ReplicatedMergeTreeQuorumEntry quorum_entry(value); if (quorum_entry.part_name != part_name) { /// The quorum has already been achieved. Moreover, another INSERT with a quorum has already started. @@ -3082,7 +3069,7 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name) Coordination::Requests ops; Coordination::Responses responses; - if (!quorum_parallel) + if (!is_parallel) { Coordination::Stat added_parts_stat; String old_added_parts = zookeeper->get(quorum_last_part_path, &added_parts_stat); @@ -3332,18 +3319,23 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. */ if (quorum) - updateQuorum(part_name); + { + /// Check if this quorum insert is parallel or not + if (zookeeper->exists(zookeeper_path + "/quorum/status")) + updateQuorum(part_name, false); + else if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + updateQuorum(part_name, true); + } - /// alexelex: i'm not sure in witch order should that and merge_selecting_task->schedule() be /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before - if (part_info.max_block != part_info.min_block) + if (part_info.level != 0 || part_info.mutation != 0) { Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); for (const String & quorum_part : quorum_parts) { auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); if (part_info.contains(quorum_part_info)) - updateQuorum(quorum_part); + updateQuorum(quorum_part, true); } } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 2bc9265331d..8dbeaabce2c 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -492,7 +492,7 @@ private: /// With the quorum being tracked, add a replica to the quorum for the part. - void updateQuorum(const String & part_name); + void updateQuorum(const String & part_name, bool is_parallel); /// Deletes info from quorum/last_part node for particular partition_id. void cleanLastPartNode(const String & partition_id); From 69c126f1f12ebe27b8f48c22996e4a2a683bb40a Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 7 Oct 2020 14:35:28 +0300 Subject: [PATCH 124/441] Possibility to move part to another disk/volume if first attempt was failed. --- src/Disks/DiskDecorator.cpp | 5 ++ src/Disks/DiskDecorator.h | 1 + src/Disks/IDisk.h | 4 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 +- .../configs/config.d/instant_moves.xml | 4 ++ .../configs/config.d/part_log.xml | 8 +++ .../configs/config.d/storage_conf.xml | 11 ++++ .../test_merge_tree_s3_failover/test.py | 62 ++++++++++++++++++- 8 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml create mode 100644 tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 7f2ea58d7cf..aaa54005f6f 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -180,4 +180,9 @@ void DiskDecorator::sync(int fd) const delegate->sync(fd); } +Executor & DiskDecorator::getExecutor() +{ + return delegate->getExecutor(); +} + } diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index f1ddfff4952..ffaf0919776 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -46,6 +46,7 @@ public: void close(int fd) const override; void sync(int fd) const override; const String getType() const override { return delegate->getType(); } + Executor & getExecutor() override; protected: DiskPtr delegate; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 688c1dfad42..ac0f5a2ae8f 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -195,10 +195,10 @@ public: /// Invoked when Global Context is shutdown. virtual void shutdown() { } -private: /// Returns executor to perform asynchronous operations. - Executor & getExecutor() { return *executor; } + virtual Executor & getExecutor() { return *executor; } +private: std::unique_ptr executor; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 40f12428561..1f68c08b6e6 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -953,7 +953,10 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat String path_to_clone = storage.relative_data_path + "detached/"; if (reserved_disk->exists(path_to_clone + relative_path)) - throw Exception("Path " + fullPath(reserved_disk, path_to_clone + relative_path) + " already exists. Can not clone ", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + { + LOG_WARNING(storage.log, "Path " + fullPath(reserved_disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + reserved_disk->removeRecursive(path_to_clone + relative_path + '/'); + } reserved_disk->createDirectory(path_to_clone); volume->getDisk()->copy(getFullRelativePath(), reserved_disk, path_to_clone); diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml new file mode 100644 index 00000000000..7b68c6946ca --- /dev/null +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml @@ -0,0 +1,4 @@ + + 0.5 + 0.5 + diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml new file mode 100644 index 00000000000..fb449ee4ad5 --- /dev/null +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml @@ -0,0 +1,8 @@ + + + + system + part_log
+ 7500 +
+
diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index d4d53ab5efe..bcd5ef97a09 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -12,6 +12,7 @@ 0 + @@ -21,6 +22,16 @@ + + +
+ default +
+ + s3 + +
+
diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index 59006e2e99a..8e37164c721 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -45,7 +45,10 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) cluster.add_instance("node", - main_configs=["configs/config.d/log_conf.xml", "configs/config.d/storage_conf.xml"], + main_configs=["configs/config.d/log_conf.xml", + "configs/config.d/storage_conf.xml", + "configs/config.d/instant_moves.xml", + "configs/config.d/part_log.xml"], with_minio=True) logging.info("Starting cluster...") cluster.start() @@ -115,3 +118,60 @@ def test_write_failover(cluster, min_bytes_for_wide_part, request_count): assert node.query("CHECK TABLE s3_failover_test") == '1\n' assert node.query("SELECT * FROM s3_failover_test FORMAT Values") == data + + +# Check that second data part move is ended successfully if first attempt was failed. +def test_move_failover(cluster): + node = cluster.instances["node"] + + node.query( + """ + CREATE TABLE s3_failover_test ( + dt DateTime, + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + TTL dt + INTERVAL 3 SECOND TO VOLUME 'external' + SETTINGS storage_policy='s3_cold' + """ + ) + + # Fail a request to S3 to break first TTL move. + fail_request(cluster, 1) + + node.query("INSERT INTO s3_failover_test VALUES (now() - 2, 0, 'data'), (now() - 2, 1, 'data')") + + # Wait for part move to S3. + max_attempts = 10 + for attempt in range(max_attempts + 1): + disk = node.query("SELECT disk_name FROM system.parts WHERE table='s3_failover_test' LIMIT 1") + if disk != "s3\n": + if attempt == max_attempts: + assert disk == "s3\n", "Expected move to S3 while part still on disk " + disk + else: + time.sleep(1) + else: + break + + # Ensure part_log is created. + node.query("SYSTEM FLUSH LOGS") + + # There should be 2 attempts to move part. + assert node.query(""" + SELECT count(*) FROM system.part_log + WHERE event_type='MovePart' AND table='s3_failover_test' + """) == '2\n' + + # First attempt should be failed with expected error. + exception = node.query(""" + SELECT exception FROM system.part_log + WHERE event_type='MovePart' AND table='s3_failover_test' + ORDER BY event_time + LIMIT 1 + """) + assert exception.find("Expected Error") != -1, exception + + # Ensure data is not corrupted. + assert node.query("CHECK TABLE s3_failover_test") == '1\n' + assert node.query("SELECT id,data FROM s3_failover_test FORMAT Values") == "(0,'data'),(1,'data')" From 8f8872dc420767747450430a7963f32a3f0533ef Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Wed, 7 Oct 2020 12:24:46 +0000 Subject: [PATCH 125/441] style check --- src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 30182697b21..bf557fa49f1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -472,7 +472,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( /// Check if this quorum insert is parallel or not if (zookeeper->exists(storage.zookeeper_path + "/quorum/status")) storage.updateQuorum(part->name, false); - else if (zookeeper->exists(storage.zookeeper_path + "/quorum/parallel/" + part->name)) + else if (zookeeper->exists(storage.zookeeper_path + "/quorum/parallel/" + part->name)) storage.updateQuorum(part->name, true); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 75cdfc5607d..fef4edfffc9 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3323,7 +3323,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora /// Check if this quorum insert is parallel or not if (zookeeper->exists(zookeeper_path + "/quorum/status")) updateQuorum(part_name, false); - else if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + else if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) updateQuorum(part_name, true); } From 1de7dbafd9fbebfdfade44d2069c8073f26442a2 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Wed, 7 Oct 2020 14:54:41 +0000 Subject: [PATCH 126/441] added 3 functional tests for parallel quorum inserts --- ...eck_many_parallel_quorum_inserts.reference | 10 +++++ ...1509_check_many_parallel_quorum_inserts.sh | 38 +++++++++++++++++ ...09_check_parallel_quorum_inserts.reference | 2 + .../01509_check_parallel_quorum_inserts.sh | 37 ++++++++++++++++ ...rallel_quorum_inserts_with_merge.reference | 2 + ...heck_parallel_quorum_inserts_with_merge.sh | 42 +++++++++++++++++++ 6 files changed, 131 insertions(+) create mode 100644 tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference create mode 100755 tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh create mode 100644 tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference create mode 100755 tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh create mode 100644 tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference create mode 100755 tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference new file mode 100644 index 00000000000..52dea650ebc --- /dev/null +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference @@ -0,0 +1,10 @@ +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 +100 0 99 4950 diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh new file mode 100755 index 00000000000..f04ce0306e4 --- /dev/null +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +NUM_REPLICAS=10 + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + DROP TABLE IF EXISTS r$i; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum_many', 'r$i') ORDER BY x; + " +done + +$CLICKHOUSE_CLIENT -n -q "SYSTEM STOP REPLICATION QUEUES quorum2;" + +function thread { + for j in {0..9}; do + a=$((($1 - 1) * 10 + $j)) + $CLICKHOUSE_CLIENT --insert_quorum 5 --insert_quorum_parallel 1 --query "INSERT INTO r$1 SELECT $a" + done +} + +for i in $(seq 1 $NUM_REPLICAS); do + thread $i & +done + +wait + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + SYSTEM SYNC REPLICA r$i; + SELECT count(), min(x), max(x), sum(x) FROM r$i; + DROP TABLE IF EXISTS r$i; +" +done diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference new file mode 100644 index 00000000000..103d5e37094 --- /dev/null +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference @@ -0,0 +1,2 @@ +5 1 5 15 +5 1 5 15 diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh new file mode 100755 index 00000000000..a3f7af8e9ca --- /dev/null +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +NUM_REPLICAS=2 +NUM_INSERTS=5 + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + DROP TABLE IF EXISTS r$i; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum', 'r$i') ORDER BY x; + " +done + +$CLICKHOUSE_CLIENT -n -q "SYSTEM STOP REPLICATION QUEUES quorum2;" + +function thread { + $CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query "INSERT INTO r1 SELECT $1" +} + +for i in $(seq 1 $NUM_INSERTS); do + thread $i & +done + +$CLICKHOUSE_CLIENT -n -q "SYSTEM START REPLICATION QUEUES quorum2;" + +wait + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + SELECT count(), min(x), max(x), sum(x) FROM r$i; + DROP TABLE IF EXISTS r$i; +" +done diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference new file mode 100644 index 00000000000..103d5e37094 --- /dev/null +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference @@ -0,0 +1,2 @@ +5 1 5 15 +5 1 5 15 diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh new file mode 100755 index 00000000000..ea07c5c410b --- /dev/null +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +NUM_REPLICAS=2 + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + DROP TABLE IF EXISTS r$i; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum_merge', 'r$i') ORDER BY x; + " +done + +$CLICKHOUSE_CLIENT -n -q " + SYSTEM STOP REPLICATION QUEUES quorum2; +" + +function thread { + $CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query "INSERT INTO r1 SELECT $1" +} + +for i in {1..5}; do + thread $i & +done + + +$CLICKHOUSE_CLIENT -n -q " + OPTIMIZE TABLE r1 FINAL; + SYSTEM START REPLICATION QUEUES quorum2; +" + +wait + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + SELECT count(), min(x), max(x), sum(x) FROM r$i; + DROP TABLE IF EXISTS r$i; +" +done From dd20dadd208af85d3701bbdf7393167038f6d19d Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Wed, 7 Oct 2020 15:24:03 +0000 Subject: [PATCH 127/441] functional tests fixes --- .../01509_check_many_parallel_quorum_inserts.sh | 12 +++++------- .../01509_check_parallel_quorum_inserts.sh | 4 ++-- ...01509_check_parallel_quorum_inserts_with_merge.sh | 7 ++++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh index f04ce0306e4..99dc1c5e8f5 100755 --- a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh @@ -14,17 +14,15 @@ for i in $(seq 1 $NUM_REPLICAS); do " done -$CLICKHOUSE_CLIENT -n -q "SYSTEM STOP REPLICATION QUEUES quorum2;" - function thread { - for j in {0..9}; do - a=$((($1 - 1) * 10 + $j)) - $CLICKHOUSE_CLIENT --insert_quorum 5 --insert_quorum_parallel 1 --query "INSERT INTO r$1 SELECT $a" - done + $CLICKHOUSE_CLIENT --insert_quorum 5 --insert_quorum_parallel 1 --query "INSERT INTO r$1 SELECT $2" } for i in $(seq 1 $NUM_REPLICAS); do - thread $i & + for j in {0..9}; do + a=$((($i - 1) * 10 + $j)) + thread $i $a & + done done wait diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh index a3f7af8e9ca..eecb06bda5d 100755 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh @@ -15,7 +15,7 @@ for i in $(seq 1 $NUM_REPLICAS); do " done -$CLICKHOUSE_CLIENT -n -q "SYSTEM STOP REPLICATION QUEUES quorum2;" +$CLICKHOUSE_CLIENT -n -q "SYSTEM STOP REPLICATION QUEUES r2;" function thread { $CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query "INSERT INTO r1 SELECT $1" @@ -25,7 +25,7 @@ for i in $(seq 1 $NUM_INSERTS); do thread $i & done -$CLICKHOUSE_CLIENT -n -q "SYSTEM START REPLICATION QUEUES quorum2;" +$CLICKHOUSE_CLIENT -n -q "SYSTEM START REPLICATION QUEUES r2;" wait diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh index ea07c5c410b..ca7db4c8d81 100755 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh @@ -6,6 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh NUM_REPLICAS=2 +NUM_INSERTS=5 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " @@ -15,21 +16,21 @@ for i in $(seq 1 $NUM_REPLICAS); do done $CLICKHOUSE_CLIENT -n -q " - SYSTEM STOP REPLICATION QUEUES quorum2; + SYSTEM STOP REPLICATION QUEUES r2; " function thread { $CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query "INSERT INTO r1 SELECT $1" } -for i in {1..5}; do +for i in $(seq 1 $NUM_INSERTS); do thread $i & done $CLICKHOUSE_CLIENT -n -q " OPTIMIZE TABLE r1 FINAL; - SYSTEM START REPLICATION QUEUES quorum2; + SYSTEM START REPLICATION QUEUES r2; " wait From ac102540d06de0775ed0d75da48ba4e1c20f54ca Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 30 Sep 2020 00:32:41 +0800 Subject: [PATCH 128/441] Global WITH and WITH propagation. --- src/Core/Settings.h | 1 + src/Interpreters/ApplyWithAliasVisitor.cpp | 53 +++++++++++++++++++ src/Interpreters/ApplyWithAliasVisitor.h | 24 +++++++++ src/Interpreters/ApplyWithGlobalVisitor.cpp | 53 +++++++++++++++++++ src/Interpreters/ApplyWithGlobalVisitor.h | 14 +++++ src/Interpreters/ApplyWithSubqueryVisitor.cpp | 22 ++++---- src/Interpreters/ApplyWithSubqueryVisitor.h | 2 - src/Interpreters/InterpreterSelectQuery.cpp | 3 ++ .../InterpreterSelectWithUnionQuery.cpp | 4 ++ src/Interpreters/ya.make | 2 + ...with_global_and_with_propagation.reference | 7 +++ ...01515_with_global_and_with_propagation.sql | 6 +++ 12 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 src/Interpreters/ApplyWithAliasVisitor.cpp create mode 100644 src/Interpreters/ApplyWithAliasVisitor.h create mode 100644 src/Interpreters/ApplyWithGlobalVisitor.cpp create mode 100644 src/Interpreters/ApplyWithGlobalVisitor.h create mode 100644 tests/queries/0_stateless/01515_with_global_and_with_propagation.reference create mode 100644 tests/queries/0_stateless/01515_with_global_and_with_propagation.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 6d3437defb2..23dd951aff8 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -471,6 +471,7 @@ class IColumn; M(Bool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \ + M(Bool, with_global, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \ #define LIST_OF_SETTINGS(M) \ COMMON_SETTINGS(M) \ diff --git a/src/Interpreters/ApplyWithAliasVisitor.cpp b/src/Interpreters/ApplyWithAliasVisitor.cpp new file mode 100644 index 00000000000..ab1656a5dde --- /dev/null +++ b/src/Interpreters/ApplyWithAliasVisitor.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +namespace DB +{ +void ApplyWithAliasVisitor::visit(ASTPtr & ast, const Data & data) +{ + if (auto * node_select = ast->as()) + { + std::optional new_data; + if (auto with = node_select->with()) + { + for (auto & child : with->children) + visit(child, data); + + std::set current_names; + for (auto & child : with->children) + { + if (auto * ast_with_alias = dynamic_cast(child.get())) + { + if (!new_data) + new_data = data; + new_data->exprs[ast_with_alias->alias] = child; + current_names.insert(ast_with_alias->alias); + } + } + for (const auto & with_alias : data.exprs) + { + if (!current_names.count(with_alias.first)) + with->children.push_back(with_alias.second->clone()); + } + } + else if (!data.exprs.empty()) + { + auto with_expression_list = std::make_shared(); + for (const auto & with_alias : data.exprs) + with_expression_list->children.push_back(with_alias.second->clone()); + node_select->setExpression(ASTSelectQuery::Expression::WITH, std::move(with_expression_list)); + } + for (auto & child : node_select->children) + { + if (child != node_select->with()) + visit(child, new_data ? *new_data : data); + } + } + else + for (auto & child : ast->children) + visit(child, data); +} + +} diff --git a/src/Interpreters/ApplyWithAliasVisitor.h b/src/Interpreters/ApplyWithAliasVisitor.h new file mode 100644 index 00000000000..8ebe8a66182 --- /dev/null +++ b/src/Interpreters/ApplyWithAliasVisitor.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include + +namespace DB +{ +/// Propagate every WITH alias expression to its descendent subqueries, with correct scoping visibility. +class ApplyWithAliasVisitor +{ +public: + struct Data + { + std::map exprs; + }; + + static void visit(ASTPtr & ast) { visit(ast, {}); } + +private: + static void visit(ASTPtr & ast, const Data & data); +}; + +} diff --git a/src/Interpreters/ApplyWithGlobalVisitor.cpp b/src/Interpreters/ApplyWithGlobalVisitor.cpp new file mode 100644 index 00000000000..70256adaaa2 --- /dev/null +++ b/src/Interpreters/ApplyWithGlobalVisitor.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include + +namespace DB +{ +void ApplyWithGlobalVisitor::visit(ASTPtr & ast) +{ + if (ASTSelectWithUnionQuery * node_union = ast->as()) + { + auto & first_select = node_union->list_of_selects->children[0]->as(); + ASTPtr with_expression_list = first_select.with(); + + if (with_expression_list) + { + std::map exprs; + for (auto & child : with_expression_list->children) + { + if (auto * ast_with_alias = dynamic_cast(child.get())) + exprs[ast_with_alias->alias] = child; + } + for (auto it = node_union->list_of_selects->children.begin() + 1; it != node_union->list_of_selects->children.end(); ++it) + { + auto & select = (*it)->as(); + auto with = select.with(); + if (with) + { + std::set current_names; + for (auto & child : with->children) + { + if (auto * ast_with_alias = dynamic_cast(child.get())) + current_names.insert(ast_with_alias->alias); + } + for (auto & with_alias : exprs) + { + if (!current_names.count(with_alias.first)) + with->children.push_back(with_alias.second->clone()); + } + } + else + select.setExpression(ASTSelectQuery::Expression::WITH, with_expression_list->clone()); + } + } + } + + for (auto & child : ast->children) + visit(child); +} + +} diff --git a/src/Interpreters/ApplyWithGlobalVisitor.h b/src/Interpreters/ApplyWithGlobalVisitor.h new file mode 100644 index 00000000000..c4dd2affd20 --- /dev/null +++ b/src/Interpreters/ApplyWithGlobalVisitor.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace DB +{ +/// Pull out the WITH statement from the first child of ASTSelectWithUnion query if any. +class ApplyWithGlobalVisitor +{ +public: + static void visit(ASTPtr & ast); +}; + +} diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index e03682dafb3..805f425beac 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -13,9 +13,8 @@ void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) { if (auto * node_select = ast->as()) { - auto with = node_select->with(); std::optional new_data; - if (with) + if (auto with = node_select->with()) { for (auto & child : with->children) visit(child, data); @@ -32,18 +31,19 @@ void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) for (auto & child : node_select->children) { - if (child != with) + if (child != node_select->with()) visit(child, new_data ? *new_data : data); } - return; } - - for (auto & child : ast->children) - visit(child, data); - if (auto * node_func = ast->as()) - visit(*node_func, data); - else if (auto * node_table = ast->as()) - visit(*node_table, data); + else + { + for (auto & child : ast->children) + visit(child, data); + if (auto * node_func = ast->as()) + visit(*node_func, data); + else if (auto * node_table = ast->as()) + visit(*node_table, data); + } } void ApplyWithSubqueryVisitor::visit(ASTTableExpression & table, const Data & data) diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.h b/src/Interpreters/ApplyWithSubqueryVisitor.h index 2aecd6aee01..ed3f9fbcd3e 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.h +++ b/src/Interpreters/ApplyWithSubqueryVisitor.h @@ -6,8 +6,6 @@ namespace DB { -// TODO After we support `union_with_global`, this visitor should also be extended to match ASTSelectQueryWithUnion. -class ASTSelectQuery; class ASTFunction; struct ASTTableExpression; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 556070d0360..53bdafabcb9 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -245,6 +246,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( source_header = input_pipe->getHeader(); } + if (context->getSettingsRef().with_global) + ApplyWithAliasVisitor().visit(query_ptr); ApplyWithSubqueryVisitor().visit(query_ptr); JoinedTables joined_tables(getSubqueryContext(*context), getSelectQuery()); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index ba0ebfaaf27..8a94e83b997 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,9 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_selects) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); + if (context->getSettingsRef().with_global) + ApplyWithGlobalVisitor().visit(query_ptr); + /// Initialize interpreters for each SELECT query. /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index bd16843517f..411100b0b9d 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -23,6 +23,8 @@ SRCS( addTypeConversionToAST.cpp AggregateDescription.cpp Aggregator.cpp + ApplyWithAliasVisitor.cpp + ApplyWithGlobalVisitor.cpp ApplyWithSubqueryVisitor.cpp ArithmeticOperationsInAgrFuncOptimize.cpp ArrayJoinAction.cpp diff --git a/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference b/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference new file mode 100644 index 00000000000..311e8f318d9 --- /dev/null +++ b/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference @@ -0,0 +1,7 @@ +1 +1 +2 1 +1 +1 +1 +2 diff --git a/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql b/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql new file mode 100644 index 00000000000..fb6b91dd7b9 --- /dev/null +++ b/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql @@ -0,0 +1,6 @@ +SET with_global = true; +WITH 1 AS x SELECT x; +WITH 1 AS x SELECT * FROM (SELECT x); +WITH 1 AS x SELECT *, x FROM (WITH 2 AS x SELECT x AS y); +WITH 1 AS x SELECT x UNION ALL SELECT x; +WITH 1 AS x SELECT x UNION ALL WITH 2 AS x SELECT x; From c9ab87ed1734237c7edd2cfb939f61391d3dc090 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 8 Oct 2020 17:06:04 +0800 Subject: [PATCH 129/441] Fix explain syntax --- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- .../InterpreterSelectWithUnionQuery.cpp | 4 --- src/Interpreters/executeQuery.cpp | 16 +++++++++-- ...with_global_and_with_propagation.reference | 28 +++++++++++++++++++ ...01515_with_global_and_with_propagation.sql | 9 +++++- 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 23dd951aff8..abf44588f0e 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -471,7 +471,7 @@ class IColumn; M(Bool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \ - M(Bool, with_global, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \ + M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \ #define LIST_OF_SETTINGS(M) \ COMMON_SETTINGS(M) \ diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 53bdafabcb9..8422b4c9878 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -246,7 +246,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( source_header = input_pipe->getHeader(); } - if (context->getSettingsRef().with_global) + if (context->getSettingsRef().enable_global_with_statement) ApplyWithAliasVisitor().visit(query_ptr); ApplyWithSubqueryVisitor().visit(query_ptr); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 8a94e83b997..ba0ebfaaf27 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -39,9 +38,6 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( if (!num_selects) throw Exception("Logical error: no children in ASTSelectWithUnionQuery", ErrorCodes::LOGICAL_ERROR); - if (context->getSettingsRef().with_global) - ApplyWithGlobalVisitor().visit(query_ptr); - /// Initialize interpreters for each SELECT query. /// Note that we pass 'required_result_column_names' to first SELECT. /// And for the rest, we pass names at the corresponding positions of 'required_result_column_names' in the result of first SELECT, diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index c82748eadc0..d7191a83971 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -337,16 +338,25 @@ static std::tuple executeQueryImpl( try { + bool ast_modified = false; /// Replace ASTQueryParameter with ASTLiteral for prepared statements. if (context.hasQueryParameters()) { ReplaceQueryParameterVisitor visitor(context.getQueryParameters()); visitor.visit(ast); - - /// Get new query after substitutions. - query = serializeAST(*ast); + ast_modified = true; } + /// Propagate WITH statement to children ASTSelect. + if (settings.enable_global_with_statement) + { + ApplyWithGlobalVisitor().visit(ast); + ast_modified = true; + } + + if (ast_modified) + query = serializeAST(*ast); + query_for_logging = prepareQueryForLogging(query, context); logQuery(query_for_logging, context, internal); diff --git a/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference b/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference index 311e8f318d9..154e60cb31b 100644 --- a/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference +++ b/tests/queries/0_stateless/01515_with_global_and_with_propagation.reference @@ -5,3 +5,31 @@ 1 1 2 +WITH 1 AS x +SELECT x +WITH 1 AS x +SELECT x +FROM +( + WITH 1 AS x + SELECT x +) +WITH 1 AS x +SELECT + y, + x +FROM +( + WITH 2 AS x + SELECT 2 AS y +) +WITH 1 AS x +SELECT x +UNION ALL +WITH 1 AS x +SELECT x +WITH 1 AS x +SELECT x +UNION ALL +WITH 2 AS x +SELECT x diff --git a/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql b/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql index fb6b91dd7b9..3257a580747 100644 --- a/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql +++ b/tests/queries/0_stateless/01515_with_global_and_with_propagation.sql @@ -1,6 +1,13 @@ -SET with_global = true; +SET enable_global_with_statement = true; + WITH 1 AS x SELECT x; WITH 1 AS x SELECT * FROM (SELECT x); WITH 1 AS x SELECT *, x FROM (WITH 2 AS x SELECT x AS y); WITH 1 AS x SELECT x UNION ALL SELECT x; WITH 1 AS x SELECT x UNION ALL WITH 2 AS x SELECT x; + +EXPLAIN SYNTAX WITH 1 AS x SELECT x; +EXPLAIN SYNTAX WITH 1 AS x SELECT * FROM (SELECT x); +EXPLAIN SYNTAX WITH 1 AS x SELECT *, x FROM (WITH 2 AS x SELECT x AS y); +EXPLAIN SYNTAX WITH 1 AS x SELECT x UNION ALL SELECT x; +EXPLAIN SYNTAX WITH 1 AS x SELECT x UNION ALL WITH 2 AS x SELECT x; From 867216103f92fd60f566610e829146c694349147 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 21 Sep 2020 18:13:01 +0800 Subject: [PATCH 130/441] Extend trivial count optimization. --- src/Interpreters/InterpreterSelectQuery.cpp | 13 +++++- src/Interpreters/TreeRewriter.cpp | 20 ++++++++- src/Storages/IStorage.h | 3 ++ src/Storages/MergeTree/KeyCondition.cpp | 33 +++++++++++--- src/Storages/MergeTree/KeyCondition.h | 12 ++++- src/Storages/StorageMergeTree.cpp | 33 ++++++++++++++ src/Storages/StorageMergeTree.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 30 +++++++++++++ src/Storages/StorageReplicatedMergeTree.h | 1 + .../00636_partition_key_parts_pruning.sh | 14 +++--- ...l_count_with_partition_predicate.reference | 9 ++++ ...trivial_count_with_partition_predicate.sql | 45 +++++++++++++++++++ 12 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference create mode 100644 tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 823808759a2..78223e56d2a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1115,6 +1115,7 @@ void InterpreterSelectQuery::executeFetchColumns( /// Optimization for trivial query like SELECT count() FROM table. bool optimize_trivial_count = syntax_analyzer_result->optimize_trivial_count + && (settings.max_parallel_replicas <= 1) && storage && !filter_info && processing_stage == QueryProcessingStage::FetchColumns @@ -1126,7 +1127,17 @@ void InterpreterSelectQuery::executeFetchColumns( { const auto & desc = query_analyzer->aggregates()[0]; const auto & func = desc.function; - std::optional num_rows = storage->totalRows(); + std::optional num_rows{}; + if (!query.prewhere() && !query.where()) + num_rows = storage->totalRows(); + else // It's possible to optimize count() given only partition predicates + { + SelectQueryInfo temp_query_info; + temp_query_info.query = query_ptr; + temp_query_info.syntax_analyzer_result = syntax_analyzer_result; + temp_query_info.sets = query_analyzer->getPreparedSets(); + num_rows = storage->totalRowsByPartitionPredicate(temp_query_info, *context); + } if (num_rows) { AggregateFunctionCount & agg_count = static_cast(*func); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 0b2f8ac3eb7..b45533f7a7b 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -478,6 +478,24 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select /// If we have no information about columns sizes, choose a column of minimum size of its data type. required.insert(ExpressionActions::getSmallestColumn(source_columns)); } + else if (is_select && metadata_snapshot) + { + const auto & partition_desc = metadata_snapshot->getPartitionKey(); + if (partition_desc.expression) + { + const auto & partition_source_columns = partition_desc.expression->getRequiredColumns(); + optimize_trivial_count = true; + for (const auto & required_column : required) + { + if (std::find(partition_source_columns.begin(), partition_source_columns.end(), required_column) + == partition_source_columns.end()) + { + optimize_trivial_count = false; + break; + } + } + } + } NameSet unknown_required_source_columns = required; @@ -620,7 +638,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( if (result.optimize_trivial_count) result.optimize_trivial_count = settings.optimize_trivial_count_query && - !select_query->where() && !select_query->prewhere() && !select_query->groupBy() && !select_query->having() && + !select_query->groupBy() && !select_query->having() && !select_query->sampleSize() && !select_query->sampleOffset() && !select_query->final() && (tables_with_columns.size() < 2 || isLeft(result.analyzed_join->kind())); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 4a2e70aa84b..836e2d7dcf1 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -463,6 +463,9 @@ public: /// Does takes underlying Storage (if any) into account. virtual std::optional totalRows() const { return {}; } + /// Same as above but also take partition predicate into account. + virtual std::optional totalRowsByPartitionPredicate(const SelectQueryInfo &, const Context &) const { return {}; } + /// If it is possible to quickly determine exact number of bytes for the table on storage: /// - memory (approximated, resident) /// - disk (compressed) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index bd45d970a7c..aae5cf3becf 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -368,8 +368,10 @@ KeyCondition::KeyCondition( const SelectQueryInfo & query_info, const Context & context, const Names & key_column_names, - const ExpressionActionsPtr & key_expr_) - : key_expr(key_expr_), prepared_sets(query_info.sets) + const ExpressionActionsPtr & key_expr_, + bool single_point_, + bool strict_) + : key_expr(key_expr_), prepared_sets(query_info.sets), single_point(single_point_), strict(strict_) { for (size_t i = 0, size = key_column_names.size(); i < size; ++i) { @@ -551,6 +553,18 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( Field & out_value, DataTypePtr & out_type) { + /// We don't look for inversed key transformations when strict is true, which is required for trivial count(). + /// Consider the following test case: + /// + /// create table test1(p DateTime, k int) engine MergeTree partition by toDate(p) order by k; + /// insert into test1 values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3); + /// select count() from test1 where p > toDateTime('2020-09-01 10:00:00'); + /// + /// toDate(DateTime) is always monotonic, but we cannot relaxing the predicates to be + /// >= toDate(toDateTime('2020-09-01 10:00:00')), which returns 3 instead of the right count: 2. + if (strict) + return false; + String expr_name = node->getColumnName(); const auto & sample_block = key_expr->getSampleBlock(); if (!sample_block.has(expr_name)) @@ -734,7 +748,8 @@ bool KeyCondition::isKeyPossiblyWrappedByMonotonicFunctions( arguments.push_back({ nullptr, key_column_type, "" }); auto func = func_builder->build(arguments); - if (!func || !func->hasInformationAboutMonotonicity()) + /// If we know the given range only contains one value, then we treat all functions as positive monotonic. + if (!func || (!single_point && !func->hasInformationAboutMonotonicity())) return false; key_column_type = func->getReturnType(); @@ -1163,13 +1178,16 @@ BoolMask KeyCondition::checkInRange( std::optional KeyCondition::applyMonotonicFunctionsChainToRange( Range key_range, const MonotonicFunctionsChain & functions, - DataTypePtr current_type) + DataTypePtr current_type, + bool single_point) { for (const auto & func : functions) { /// We check the monotonicity of each function on a specific range. - IFunction::Monotonicity monotonicity = func->getMonotonicityForRange( - *current_type.get(), key_range.left, key_range.right); + /// If we know the given range only contains one value, then we treat all functions as positive monotonic. + IFunction::Monotonicity monotonicity = single_point + ? IFunction::Monotonicity{true} + : func->getMonotonicityForRange(*current_type.get(), key_range.left, key_range.right); if (!monotonicity.is_monotonic) { @@ -1299,7 +1317,8 @@ BoolMask KeyCondition::checkInHyperrectangle( std::optional new_range = applyMonotonicFunctionsChainToRange( *key_range, element.monotonic_functions_chain, - data_types[element.key_column] + data_types[element.key_column], + single_point ); if (!new_range) diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index fdae7335646..04591f197bb 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -232,7 +232,9 @@ public: const SelectQueryInfo & query_info, const Context & context, const Names & key_column_names, - const ExpressionActionsPtr & key_expr); + const ExpressionActionsPtr & key_expr, + bool single_point_ = false, + bool strict_ = false); /// Whether the condition and its negation are feasible in the direct product of single column ranges specified by `hyperrectangle`. BoolMask checkInHyperrectangle( @@ -307,7 +309,8 @@ public: static std::optional applyMonotonicFunctionsChainToRange( Range key_range, const MonotonicFunctionsChain & functions, - DataTypePtr current_type); + DataTypePtr current_type, + bool single_point = false); bool matchesExactContinuousRange() const; @@ -413,6 +416,11 @@ private: ColumnIndices key_columns; ExpressionActionsPtr key_expr; PreparedSets prepared_sets; + + // If true, always allow key_expr to be wrapped by function + bool single_point; + // If true, do not use always_monotonic information to transform constants + bool strict; }; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 55fb42b550e..e2d5daecada 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -189,6 +189,39 @@ std::optional StorageMergeTree::totalRows() const return getTotalActiveSizeInRows(); } +std::optional StorageMergeTree::totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + const auto & partition_key = metadata_snapshot->getPartitionKey(); + Names partition_key_columns = partition_key.column_names; + KeyCondition key_condition( + query_info, context, partition_key_columns, partition_key.expression, true /* single_point */, true /* strict */); + if (key_condition.alwaysUnknownOrTrue()) + return {}; + std::unordered_map partition_filter_map; + size_t res = 0; + auto lock = lockParts(); + for (const auto & part : getDataPartsStateRange(DataPartState::Committed)) + { + if (part->isEmpty()) + continue; + const auto & partition_id = part->info.partition_id; + bool is_valid; + if (auto it = partition_filter_map.find(partition_id); it != partition_filter_map.end()) + is_valid = it->second; + else + { + const auto & partition_value = part->partition.value; + std::vector index_value(partition_value.begin(), partition_value.end()); + is_valid = key_condition.mayBeTrueInRange(partition_value.size(), index_value.data(), index_value.data(), partition_key.data_types); + partition_filter_map.emplace(partition_id, is_valid); + } + if (is_valid) + res += part->rows_count; + } + return res; +} + std::optional StorageMergeTree::totalBytes() const { return getTotalActiveSizeInBytes(); diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 5662f9e0088..7d092602703 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -47,6 +47,7 @@ public: unsigned num_streams) override; std::optional totalRows() const override; + std::optional totalRowsByPartitionPredicate(const SelectQueryInfo &, const Context &) const override; std::optional totalBytes() const override; BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9613bd5111d..9bd749a32ff 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3563,6 +3563,36 @@ std::optional StorageReplicatedMergeTree::totalRows() const return res; } +std::optional StorageReplicatedMergeTree::totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + const auto & partition_key = metadata_snapshot->getPartitionKey(); + Names partition_key_columns = partition_key.column_names; + KeyCondition key_condition( + query_info, context, partition_key_columns, partition_key.expression, true /* single_point */, true /* strict */); + if (key_condition.alwaysUnknownOrTrue()) + return {}; + std::unordered_map partition_filter_map; + size_t res = 0; + foreachCommittedParts([&](auto & part) + { + const auto & partition_id = part->info.partition_id; + bool is_valid; + if (auto it = partition_filter_map.find(partition_id); it != partition_filter_map.end()) + is_valid = it->second; + else + { + const auto & partition_value = part->partition.value; + std::vector index_value(partition_value.begin(), partition_value.end()); + is_valid = key_condition.mayBeTrueInRange(partition_value.size(), index_value.data(), index_value.data(), partition_key.data_types); + partition_filter_map.emplace(partition_id, is_valid); + } + if (is_valid) + res += part->rows_count; + }); + return res; +} + std::optional StorageReplicatedMergeTree::totalBytes() const { UInt64 res = 0; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index d851082d5c2..1ed2066f221 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -97,6 +97,7 @@ public: unsigned num_streams) override; std::optional totalRows() const override; + std::optional totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const override; std::optional totalBytes() const override; BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh index aad54fc0a73..8150d52abc9 100755 --- a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh +++ b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh @@ -30,14 +30,14 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO composite_partition_key VALUES \ ${CLICKHOUSE_CLIENT} --query="INSERT INTO composite_partition_key VALUES \ (301, 20, 3), (302, 21, 3), (303, 22, 3)" -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a > 400 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b = 11 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE c = 4 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a > 400 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b = 11 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE c = 4 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a < 200 AND c = 2 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b < 20 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b >= 12 AND c = 2 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a < 200 AND c = 2 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b < 20 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b >= 12 AND c = 2 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b = 21 AND c = 3 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b = 21 AND c = 3 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' ${CLICKHOUSE_CLIENT} --query="DROP TABLE composite_partition_key" diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference new file mode 100644 index 00000000000..4fe8dbf8cfb --- /dev/null +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference @@ -0,0 +1,9 @@ +0 +0 +2 +1 +1 +0 +2 +0 +3 diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql new file mode 100644 index 00000000000..110e883803d --- /dev/null +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql @@ -0,0 +1,45 @@ +drop table if exists test1; + +create table test1(p DateTime, k int) engine MergeTree partition by toDate(p) order by k; +insert into test1 values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3); + +set max_rows_to_read = 1; +-- non-optimized +select count() from test1 settings max_parallel_replicas = 3; -- { serverError 158; } +-- optimized (toYear is monotonic and we provide the partition expr as is) +select count() from test1 where toYear(toDate(p)) = 1999; +-- non-optimized (toDate(DateTime) is always monotonic, but we cannot relaxing the predicates to do trivial count()) +select count() from test1 where p > toDateTime('2020-09-01 10:00:00'); -- { serverError 158; } +-- optimized (partition expr wrapped with non-monotonic functions) +select count() FROM test1 where toDate(p) = '2020-09-01' and sipHash64(toString(toDate(p))) % 2 = 1; +select count() FROM test1 where toDate(p) = '2020-09-01' and sipHash64(toString(toDate(p))) % 2 = 0; +-- non-optimized (some predicate depends on non-partition_expr columns) +select count() FROM test1 where toDate(p) = '2020-09-01' and k = 2; -- { serverError 158; } +-- optimized +select count() from test1 where toDate(p) > '2020-09-01'; + +create table test_tuple(p DateTime, i int, j int) engine MergeTree partition by (toDate(p), i) order by j; + +insert into test_tuple values ('2020-09-01 00:01:02', 1, 2), ('2020-09-01 00:01:03', 2, 3), ('2020-09-02 00:01:03', 3, 4); + +-- optimized +select count() from test_tuple where toDate(p) > '2020-09-01'; +-- optimized +select count() from test_tuple where toDate(p) > '2020-09-01' and i = 1; +-- optimized +select count() from test_tuple where i > 1; +-- optimized +select count() from test_tuple where i < 1; + +create table test_two_args(i int, j int, k int) engine MergeTree partition by i + j order by k; + +insert into test_two_args values (1, 2, 3), (2, 1, 3), (0, 3, 4); + +-- optimized +select count() from test_two_args where i + j = 3; +-- non-optimized +select count() from test_two_args where i = 1; -- { serverError 158; } + +drop table test1; +drop table test_tuple; +drop table test_two_args; From 1a6d2bc53812dbf92dc5b781152165b21f630af6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 8 Oct 2020 15:46:28 +0300 Subject: [PATCH 131/441] fix prewhere with virtuals columns --- .../MergeTree/MergeTreeBaseSelectProcessor.cpp | 3 --- .../01511_prewhere_with_virtuals.reference | 3 +++ .../0_stateless/01511_prewhere_with_virtuals.sql | 12 ++++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01511_prewhere_with_virtuals.reference create mode 100644 tests/queries/0_stateless/01511_prewhere_with_virtuals.sql diff --git a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 8cb24bb0cd6..c852151f27d 100644 --- a/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -330,9 +330,6 @@ void MergeTreeBaseSelectProcessor::executePrewhereActions(Block & block, const P auto & ctn = block.getByName(prewhere_info->prewhere_column_name); ctn.column = ctn.type->createColumnConst(block.rows(), 1u)->convertToFullColumnIfConst(); } - - if (!block) - block.insert({nullptr, std::make_shared(), "_nothing"}); } } diff --git a/tests/queries/0_stateless/01511_prewhere_with_virtuals.reference b/tests/queries/0_stateless/01511_prewhere_with_virtuals.reference new file mode 100644 index 00000000000..bd19df3c15d --- /dev/null +++ b/tests/queries/0_stateless/01511_prewhere_with_virtuals.reference @@ -0,0 +1,3 @@ +1_2_2_0 4 +2_3_3_0 3 +0_1_1_0 diff --git a/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql b/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql new file mode 100644 index 00000000000..8b272c8cff1 --- /dev/null +++ b/tests/queries/0_stateless/01511_prewhere_with_virtuals.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS test_not_found_column_nothing; + +CREATE TABLE test_not_found_column_nothing +( + col001 UInt8, + col002 UInt8 +) Engine=MergeTree ORDER BY tuple() PARTITION BY col001 % 3; + +INSERT INTO test_not_found_column_nothing(col001) SELECT number FROM numbers(11); + +SELECT _part, count() FROM test_not_found_column_nothing PREWHERE col001 % 3 != 0 GROUP BY _part ORDER BY _part; +SELECT _part FROM test_not_found_column_nothing PREWHERE col001 = 0; From c6d8b94ad63de15eebb5b12b0943e28a15735134 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 8 Oct 2020 16:14:44 +0300 Subject: [PATCH 132/441] Add some tests --- .../test_quorum_inserts_parallel/__init__.py | 1 + .../test_quorum_inserts_parallel/test.py | 96 +++++++++++++++++++ ...rallel_quorum_insert_no_replicas.reference | 12 +++ ...509_parallel_quorum_insert_no_replicas.sql | 66 +++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 tests/integration/test_quorum_inserts_parallel/__init__.py create mode 100644 tests/integration/test_quorum_inserts_parallel/test.py create mode 100644 tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.reference create mode 100644 tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.sql diff --git a/tests/integration/test_quorum_inserts_parallel/__init__.py b/tests/integration/test_quorum_inserts_parallel/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_quorum_inserts_parallel/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_quorum_inserts_parallel/test.py b/tests/integration/test_quorum_inserts_parallel/test.py new file mode 100644 index 00000000000..57bb95b7113 --- /dev/null +++ b/tests/integration/test_quorum_inserts_parallel/test.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import pytest +from helpers.cluster import ClickHouseCluster +from multiprocessing.dummy import Pool +from helpers.network import PartitionManager +from helpers.client import QueryRuntimeException +from helpers.test_tools import assert_eq_with_retry + + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance("node1", with_zookeeper=True) +node2 = cluster.add_instance("node2", with_zookeeper=True) +node3 = cluster.add_instance("node3", with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + global cluster + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_parallel_quorum_actually_parallel(started_cluster): + settings = {"insert_quorum": "3", "insert_quorum_parallel": "1"} + for i, node in enumerate([node1, node2, node3]): + node.query("CREATE TABLE r (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/r', '{num}') ORDER BY tuple()".format(num=i)) + + p = Pool(10) + + def long_insert(node): + node.query("INSERT INTO r SELECT number, toString(number) FROM numbers(5) where sleepEachRow(1) == 0", settings=settings) + + job = p.apply_async(long_insert, (node1,)) + + node2.query("INSERT INTO r VALUES (6, '6')", settings=settings) + assert node1.query("SELECT COUNT() FROM r") == "1\n" + assert node2.query("SELECT COUNT() FROM r") == "1\n" + assert node3.query("SELECT COUNT() FROM r") == "1\n" + + node1.query("INSERT INTO r VALUES (7, '7')", settings=settings) + assert node1.query("SELECT COUNT() FROM r") == "2\n" + assert node2.query("SELECT COUNT() FROM r") == "2\n" + assert node3.query("SELECT COUNT() FROM r") == "2\n" + + job.get() + + assert node1.query("SELECT COUNT() FROM r") == "7\n" + assert node2.query("SELECT COUNT() FROM r") == "7\n" + assert node3.query("SELECT COUNT() FROM r") == "7\n" + p.close() + p.join() + + +def test_parallel_quorum_actually_quorum(started_cluster): + settings = {"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "3000"} + for i, node in enumerate([node1, node2, node3]): + node.query("CREATE TABLE q (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/q', '{num}') ORDER BY tuple()".format(num=i)) + + with PartitionManager() as pm: + pm.partition_instances(node2, node1, port=9009) + pm.partition_instances(node2, node3, port=9009) + with pytest.raises(QueryRuntimeException): + node1.query("INSERT INTO q VALUES(1, 'Hello')", settings=settings) + + assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "1") + assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "0") + assert_eq_with_retry(node3, "SELECT COUNT() FROM q", "1") + + node1.query("INSERT INTO q VALUES(2, 'wlrd')", settings={"insert_quorum": "2", "insert_quorum_parallel": "1", "insert_quorum_timeout": "3000"}) + + assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "2") + assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "0") + assert_eq_with_retry(node3, "SELECT COUNT() FROM q", "2") + + def insert_value_to_node(node, settings): + node.query("INSERT INTO q VALUES(3, 'Hi')", settings=settings) + + p = Pool(2) + res = p.apply_async(insert_value_to_node, (node1, {"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "10000"})) + insert_value_to_node(node2, {}) + + assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "3") + assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "1") + assert_eq_with_retry(node3, "SELECT COUNT() FROM q", "3") + + res.get() + p.close() + p.join() + + node2.query("SYSTEM SYNC REPLICA q", timeout=10) + assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "3") diff --git a/tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.reference b/tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.reference new file mode 100644 index 00000000000..5647890ac1c --- /dev/null +++ b/tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.reference @@ -0,0 +1,12 @@ +insert to two replicas works +1 +1 +insert to single replica works +3 +3 +deduplication works +3 +3 +insert happened +4 +4 diff --git a/tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.sql b/tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.sql new file mode 100644 index 00000000000..7607a4e9012 --- /dev/null +++ b/tests/queries/0_stateless/01509_parallel_quorum_insert_no_replicas.sql @@ -0,0 +1,66 @@ +DROP TABLE IF EXISTS r1; +DROP TABLE IF EXISTS r2; + +CREATE TABLE r1 ( + key UInt64, value String +) +ENGINE = ReplicatedMergeTree('/clickhouse/01509_no_repliacs', '1') +ORDER BY tuple(); + +CREATE TABLE r2 ( + key UInt64, value String +) +ENGINE = ReplicatedMergeTree('/clickhouse/01509_no_repliacs', '2') +ORDER BY tuple(); + +SET insert_quorum_parallel=1; + +SET insert_quorum=3; +INSERT INTO r1 VALUES(1, '1'); --{serverError 285} + +SELECT 'insert to two replicas works'; +SET insert_quorum=2, insert_quorum_parallel=1; +INSERT INTO r1 VALUES(1, '1'); + +SELECT COUNT() FROM r1; +SELECT COUNT() FROM r2; + +DETACH TABLE r2; + +INSERT INTO r1 VALUES(2, '2'); --{serverError 285} + +SET insert_quorum=1, insert_quorum_parallel=1; +SELECT 'insert to single replica works'; +INSERT INTO r1 VALUES(2, '2'); + +ATTACH TABLE r2; + +SET insert_quorum=2, insert_quorum_parallel=1; + +INSERT INTO r1 VALUES(3, '3'); + +SELECT COUNT() FROM r1; +SELECT COUNT() FROM r2; + +SELECT 'deduplication works'; +INSERT INTO r2 VALUES(3, '3'); + +SELECT COUNT() FROM r1; +SELECT COUNT() FROM r2; + +SYSTEM STOP FETCHES r2; + +SET insert_quorum_timeout=0; + +INSERT INTO r1 VALUES (4, '4'); -- { serverError 319 } + +SYSTEM START FETCHES r2; + +SYSTEM SYNC REPLICA r2; + +SELECT 'insert happened'; +SELECT COUNT() FROM r1; +SELECT COUNT() FROM r2; + +DROP TABLE IF EXISTS r1; +DROP TABLE IF EXISTS r2; From c8bb6237339f93cdfb16a7492283cbf970508b39 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 8 Oct 2020 17:35:20 +0300 Subject: [PATCH 133/441] Add test for merged part --- .../01509_parallel_quorum_and_merge.reference | 4 ++ .../01509_parallel_quorum_and_merge.sh | 66 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference create mode 100755 tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference new file mode 100644 index 00000000000..a5c40a85049 --- /dev/null +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference @@ -0,0 +1,4 @@ +all_0_1_1 +DownloadPart +2 +2 diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh new file mode 100755 index 00000000000..214c39a21cc --- /dev/null +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q2" + + +$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q1 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_q', 'r1') ORDER BY tuple() SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0" + +$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q2 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_q', 'r2') ORDER BY tuple() SETTINGS always_fetch_merged_part = 1" + +$CLICKHOUSE_CLIENT -q "SYSTEM STOP REPLICATION QUEUES parallel_q2" + +$CLICKHOUSE_CLIENT -q "INSERT INTO parallel_q1 VALUES (1)" + +$CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query="INSERT INTO parallel_q1 VALUES (2)" & + +part_count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM system.parts WHERE table='parallel_q1' and database='${CLICKHOUSE_DATABASE}'") + +# Check part inserted locally +while [[ $part_count != 2 ]] +do + sleep 0.1 + part_count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM system.parts WHERE table='parallel_q1' and database='${CLICKHOUSE_DATABASE}'") +done + +$CLICKHOUSE_CLIENT --replication_alter_partitions_sync 0 -q "OPTIMIZE TABLE parallel_q1 FINAL" + +# check part merged locally +has_part=$($CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM system.parts WHERE table='parallel_q1' and database='${CLICKHOUSE_DATABASE}' and name='all_0_1_1'") + +while [[ $has_part != 1 ]] +do + sleep 0.1 + has_part=$($CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM system.parts WHERE table='parallel_q1' and database='${CLICKHOUSE_DATABASE}' and name='all_0_1_1'") +done + +# check source parts removed locally +active_parts_count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM system.parts WHERE table='parallel_q1' and database='${CLICKHOUSE_DATABASE}' and active=1") + +while [[ $active_parts_count != 1 ]] +do + sleep 0.1 + active_parts_count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM system.parts WHERE table='parallel_q1' and database='${CLICKHOUSE_DATABASE}'") +done + +# download merged part +$CLICKHOUSE_CLIENT -q "SYSTEM START REPLICATION QUEUES parallel_q2" + +$CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA parallel_q2" + +# quorum satisfied even for merged part +wait + +$CLICKHOUSE_CLIENT --query="SYSTEM FLUSH LOGS" +$CLICKHOUSE_CLIENT --query="SELECT name FROM system.parts WHERE table='parallel_q2' and database='${CLICKHOUSE_DATABASE}' and active=1 ORDER BY name" +$CLICKHOUSE_CLIENT --query="SELECT event_type FROM system.part_log WHERE table='parallel_q2' and database='${CLICKHOUSE_DATABASE}' and part_name='all_0_1_1'" +$CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM parallel_q2" +$CLICKHOUSE_CLIENT --query="SELECT COUNT() FROM parallel_q1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q1" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q2" From 22d2251351f794fd6e8b15dfcd39a0cb2b9bb171 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Thu, 8 Oct 2020 14:47:05 +0000 Subject: [PATCH 134/441] remove old quorum_with_merge test (rc) --- ...rallel_quorum_inserts_with_merge.reference | 2 - ...heck_parallel_quorum_inserts_with_merge.sh | 43 ------------------- 2 files changed, 45 deletions(-) delete mode 100644 tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference delete mode 100755 tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference deleted file mode 100644 index 103d5e37094..00000000000 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.reference +++ /dev/null @@ -1,2 +0,0 @@ -5 1 5 15 -5 1 5 15 diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh deleted file mode 100755 index ca7db4c8d81..00000000000 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_with_merge.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../shell_config.sh - -NUM_REPLICAS=2 -NUM_INSERTS=5 - -for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " - DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum_merge', 'r$i') ORDER BY x; - " -done - -$CLICKHOUSE_CLIENT -n -q " - SYSTEM STOP REPLICATION QUEUES r2; -" - -function thread { - $CLICKHOUSE_CLIENT --insert_quorum 2 --insert_quorum_parallel 1 --query "INSERT INTO r1 SELECT $1" -} - -for i in $(seq 1 $NUM_INSERTS); do - thread $i & -done - - -$CLICKHOUSE_CLIENT -n -q " - OPTIMIZE TABLE r1 FINAL; - SYSTEM START REPLICATION QUEUES r2; -" - -wait - -for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " - SELECT count(), min(x), max(x), sum(x) FROM r$i; - DROP TABLE IF EXISTS r$i; -" -done From 6877f4aa9c8ab571d071154865fcebc3917f0fc8 Mon Sep 17 00:00:00 2001 From: Alexandra Latysheva Date: Thu, 8 Oct 2020 15:35:41 +0000 Subject: [PATCH 135/441] for parallel and non-parallel quorum insertion at the same time --- .../MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp | 6 +++--- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index bf557fa49f1..c8530943873 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -470,10 +470,10 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart( { /// We get duplicate part without fetch /// Check if this quorum insert is parallel or not - if (zookeeper->exists(storage.zookeeper_path + "/quorum/status")) - storage.updateQuorum(part->name, false); - else if (zookeeper->exists(storage.zookeeper_path + "/quorum/parallel/" + part->name)) + if (zookeeper->exists(storage.zookeeper_path + "/quorum/parallel/" + part->name)) storage.updateQuorum(part->name, true); + else if (zookeeper->exists(storage.zookeeper_path + "/quorum/status")) + storage.updateQuorum(part->name, false); } /// We are waiting for quorum to be satisfied. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index cd91f40647b..f61cb924352 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3323,10 +3323,10 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (quorum) { /// Check if this quorum insert is parallel or not - if (zookeeper->exists(zookeeper_path + "/quorum/status")) - updateQuorum(part_name, false); - else if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) updateQuorum(part_name, true); + else if (zookeeper->exists(zookeeper_path + "/quorum/status")) + updateQuorum(part_name, false); } /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before From 1103ce0d785fc9e2ba17b3cdbc15e616ad783c1a Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 8 Oct 2020 15:36:03 -0400 Subject: [PATCH 136/441] Updating exit codes and messages in failing tests. --- .../authentication/tests/server_config.py | 32 ++++++++++++------- .../ldap/authentication/tests/user_config.py | 4 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py index 5658f7a9399..80f2a496b0e 100644 --- a/tests/testflows/ldap/authentication/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -28,7 +28,8 @@ def invalid_host(self): servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -44,7 +45,8 @@ def empty_host(self): servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -60,7 +62,8 @@ def missing_host(self): servers = {"foo": {"port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -75,7 +78,8 @@ def invalid_port(self): servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server." + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -93,7 +97,8 @@ def invalid_auth_dn_prefix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -110,7 +115,8 @@ def invalid_auth_dn_suffix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -127,7 +133,8 @@ def invalid_enable_tls_value(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap1' is not configured" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -148,7 +155,8 @@ def invalid_tls_require_cert_value(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap2' is not configured" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -167,8 +175,8 @@ def empty_ca_cert_dir(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -187,8 +195,8 @@ def empty_ca_cert_file(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "Received from localhost:9000. DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain)" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) diff --git a/tests/testflows/ldap/authentication/tests/user_config.py b/tests/testflows/ldap/authentication/tests/user_config.py index 391e4ee24c5..36ed33ed17a 100644 --- a/tests/testflows/ldap/authentication/tests/user_config.py +++ b/tests/testflows/ldap/authentication/tests/user_config.py @@ -54,8 +54,8 @@ def empty_server_not_defined(self, timeout=20): "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" }} users = [{"server": "foo", "username": "user1", "password": "user1", "login": True, - "errorcode": 36, - "message": "DB::Exception: LDAP server 'foo' is not configured" + "errorcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) From a6178cc8769c758cd3215a2a3afbd27e1cd64411 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 8 Oct 2020 23:39:24 +0300 Subject: [PATCH 137/441] try fix select count() for MaterializeMySQL --- src/Storages/StorageMaterializeMySQL.cpp | 11 +++++++ src/Storages/StorageMaterializeMySQL.h | 1 + src/Storages/StorageMerge.cpp | 8 +++++ src/Storages/StorageMerge.h | 1 + .../materialize_with_ddl.py | 30 +++++++++++++++++++ .../test_materialize_mysql_database/test.py | 7 +++++ 6 files changed, 58 insertions(+) diff --git a/src/Storages/StorageMaterializeMySQL.cpp b/src/Storages/StorageMaterializeMySQL.cpp index 7d908736bdc..c56d0f610e7 100644 --- a/src/Storages/StorageMaterializeMySQL.cpp +++ b/src/Storages/StorageMaterializeMySQL.cpp @@ -105,6 +105,17 @@ NamesAndTypesList StorageMaterializeMySQL::getVirtuals() const return nested_storage->getVirtuals(); } +IStorage::ColumnSizeByName StorageMaterializeMySQL::getColumnSizes() const +{ + auto sizes = nested_storage->getColumnSizes(); + auto nested_header = nested_storage->getInMemoryMetadataPtr()->getSampleBlock(); + String sign_column_name = nested_header.getByPosition(nested_header.columns() - 2).name; + String version_column_name = nested_header.getByPosition(nested_header.columns() - 1).name; + sizes.erase(sign_column_name); + sizes.erase(version_column_name); + return sizes; +} + } #endif diff --git a/src/Storages/StorageMaterializeMySQL.h b/src/Storages/StorageMaterializeMySQL.h index 4278ce64bd7..ba79fd6bca4 100644 --- a/src/Storages/StorageMaterializeMySQL.h +++ b/src/Storages/StorageMaterializeMySQL.h @@ -26,6 +26,7 @@ public: const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; NamesAndTypesList getVirtuals() const override; + ColumnSizeByName getColumnSizes() const override; private: StoragePtr nested_storage; diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index bade0810320..5c68d2dd047 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -538,6 +538,14 @@ void StorageMerge::convertingSourceStream( } } +IStorage::ColumnSizeByName StorageMerge::getColumnSizes() const +{ + + auto first_materialize_mysql = getFirstTable([](const StoragePtr & table) { return table && table->getName() == "MaterializeMySQL"; }); + if (!first_materialize_mysql) + return {}; + return first_materialize_mysql->getColumnSizes(); +} void registerStorageMerge(StorageFactory & factory) { diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 5e2d56d18c0..78debf194f0 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -66,6 +66,7 @@ private: DatabaseTablesIteratorPtr getDatabaseIterator(const Context & context) const; NamesAndTypesList getVirtuals() const override; + ColumnSizeByName getColumnSizes() const override; protected: StorageMerge( diff --git a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py index 3990f7dbd33..b97a1563212 100644 --- a/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py +++ b/tests/integration/test_materialize_mysql_database/materialize_with_ddl.py @@ -431,3 +431,33 @@ def query_event_with_empty_transaction(clickhouse_node, mysql_node, service_name check_query(clickhouse_node, "SELECT * FROM test_database.t1 ORDER BY a FORMAT TSV", "1\tBEGIN\n2\tBEGIN\n") clickhouse_node.query("DROP DATABASE test_database") mysql_node.query("DROP DATABASE test_database") + +def select_without_columns(clickhouse_node, mysql_node, service_name): + mysql_node.query("CREATE DATABASE db") + mysql_node.query("CREATE TABLE db.t (a INT PRIMARY KEY, b INT)") + clickhouse_node.query( + "CREATE DATABASE db ENGINE = MaterializeMySQL('{}:3306', 'db', 'root', 'clickhouse')".format(service_name)) + check_query(clickhouse_node, "SHOW TABLES FROM db FORMAT TSV", "t\n") + clickhouse_node.query("SYSTEM STOP MERGES db.t") + clickhouse_node.query("CREATE VIEW v AS SELECT * FROM db.t") + mysql_node.query("INSERT INTO db.t VALUES (1, 1), (2, 2)") + mysql_node.query("DELETE FROM db.t WHERE a=2;") + check_query(clickhouse_node, "SELECT count((_sign, _version)) FROM db.t FORMAT TSV", "3\n") + + assert clickhouse_node.query("SELECT count(_sign) FROM db.t FORMAT TSV") == "2\n" + assert clickhouse_node.query("SELECT count(_version) FROM db.t FORMAT TSV") == "2\n" + + assert clickhouse_node.query("SELECT count() FROM db.t FORMAT TSV") == "1\n" + assert clickhouse_node.query("SELECT count(*) FROM db.t FORMAT TSV") == "1\n" + assert clickhouse_node.query("SELECT count() FROM (SELECT * FROM db.t) FORMAT TSV") == "1\n" + assert clickhouse_node.query("SELECT count() FROM v FORMAT TSV") == "1\n" + assert clickhouse_node.query("SELECT count() FROM merge('db', 't') FORMAT TSV") == "1\n" + assert clickhouse_node.query("SELECT count() FROM remote('localhost', 'db', 't') FORMAT TSV") == "1\n" + + assert clickhouse_node.query("SELECT _part FROM db.t FORMAT TSV") == "0_1_1_0\n" + assert clickhouse_node.query("SELECT _part FROM remote('localhost', 'db', 't') FORMAT TSV") == "0_1_1_0\n" + + + clickhouse_node.query("DROP VIEW v") + clickhouse_node.query("DROP DATABASE db") + mysql_node.query("DROP DATABASE db") diff --git a/tests/integration/test_materialize_mysql_database/test.py b/tests/integration/test_materialize_mysql_database/test.py index 237df999c62..6df831e1e7d 100644 --- a/tests/integration/test_materialize_mysql_database/test.py +++ b/tests/integration/test_materialize_mysql_database/test.py @@ -146,3 +146,10 @@ def test_materialize_database_ddl_with_empty_transaction_5_7(started_cluster, st def test_materialize_database_ddl_with_empty_transaction_8_0(started_cluster, started_mysql_8_0): materialize_with_ddl.query_event_with_empty_transaction(clickhouse_node, started_mysql_8_0, "mysql8_0") + + +def test_select_without_columns_5_7(started_cluster, started_mysql_5_7): + materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_5_7, "mysql1") + +def test_select_without_columns_8_0(started_cluster, started_mysql_8_0): + materialize_with_ddl.select_without_columns(clickhouse_node, started_mysql_8_0, "mysql8_0") From e348ec17b2ae9afb7838ad131049966c2a16e75a Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 9 Oct 2020 00:57:23 +0400 Subject: [PATCH 138/441] Refactor role handling --- src/Access/LDAPAccessStorage.cpp | 110 ++++++++++++++++--------------- src/Access/LDAPAccessStorage.h | 3 +- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index d1dbf10cfbc..a3945686ff7 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include @@ -64,54 +66,66 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m return this->processRoleChange(id, entity); } ); + + /// Update `roles_of_interests` with initial values. + for (const auto & role_name : default_role_names) + { + if (auto role_id = access_control_manager->find(role_name)) + roles_of_interest.emplace(*role_id, role_name); + } } void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) { std::scoped_lock lock(mutex); - auto role_ptr = typeid_cast>(entity); - if (role_ptr) - { - if (default_role_names.find(role_ptr->getName()) != default_role_names.end()) - { - auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr - { - auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && user_ptr->granted_roles.roles.find(id) == user_ptr->granted_roles.roles.end()) - { - auto clone = user_ptr->clone(); - auto user_clone_ptr = typeid_cast>(clone); - user_clone_ptr->granted_roles.grant(id); - return user_clone_ptr; - } - return cached_entity; - }; - memory_storage.update(memory_storage.findAll(), update_func); - roles_of_interest.insert(id); - } + /// Update `roles_of_interests`. + auto role = typeid_cast>(entity); + bool need_to_update_users = false; + + if (role && default_role_names.contains(role->getName())) + { + /// If a role was created with one of the `default_role_names` or renamed to one of the `default_role_names`, + /// then set `need_to_update_users`. + need_to_update_users = roles_of_interest.insert_or_assign(id, role->getName()).second; } else { - if (roles_of_interest.find(id) != roles_of_interest.end()) - { - auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr - { - auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && user_ptr->granted_roles.roles.find(id) != user_ptr->granted_roles.roles.end()) - { - auto clone = user_ptr->clone(); - auto user_clone_ptr = typeid_cast>(clone); - user_clone_ptr->granted_roles.revoke(id); - return user_clone_ptr; - } - return cached_entity; - }; + /// If a role was removed or renamed to a name which isn't contained in the `default_role_names`, + /// then set `need_to_update_users`. + need_to_update_users = roles_of_interest.erase(id) > 0; + } - memory_storage.update(memory_storage.findAll(), update_func); - roles_of_interest.erase(id); - } + /// Update users which have been created. + if (need_to_update_users) + { + auto update_func = [this] (const AccessEntityPtr & entity_) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + auto & granted_roles = changed_user->granted_roles.roles; + granted_roles.clear(); + boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end())); + return changed_user; + } + return entity_; + }; + memory_storage.update(memory_storage.findAll(), update_func); + } +} + + +void LDAPAccessStorage::checkAllDefaultRoleNamesFoundNoLock() const +{ + boost::container::flat_set role_names_of_interest; + boost::range::copy(roles_of_interest | boost::adaptors::map_values, std::inserter(role_names_of_interest, role_names_of_interest.end())); + + for (const auto & role_name : default_role_names) + { + if (!role_names_of_interest.contains(role_name)) + throwDefaultRoleNotFound(role_name); } } @@ -257,15 +271,10 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw if (!isAddressAllowedImpl(*user, address)) throwAddressNotAllowed(address); - for (const auto& role_name : default_role_names) - { - std::optional role_id = access_control_manager->find(role_name); - if (!role_id) - throwDefaultRoleNotFound(role_name); + checkAllDefaultRoleNamesFoundNoLock(); - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } + auto & granted_roles = user->granted_roles.roles; + boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end())); return memory_storage.insert(user); } @@ -287,15 +296,10 @@ UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); - for (const auto& role_name : default_role_names) - { - std::optional role_id = access_control_manager->find(role_name); - if (!role_id) - throwDefaultRoleNotFound(role_name); + checkAllDefaultRoleNamesFoundNoLock(); - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } + auto & granted_roles = user->granted_roles.roles; + boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end())); return memory_storage.insert(user); } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 1e6e0713568..7ac37b9142c 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -55,6 +55,7 @@ private: // IAccessStorage implementations. private: void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + void checkAllDefaultRoleNamesFoundNoLock() const; [[noreturn]] static void throwDefaultRoleNotFound(const String & role_name); @@ -62,7 +63,7 @@ private: AccessControlManager * access_control_manager = nullptr; String ldap_server; std::set default_role_names; - mutable std::set roles_of_interest; + std::map roles_of_interest; ext::scope_guard role_change_subscription; mutable MemoryAccessStorage memory_storage; }; From 3861a0392736b866ad77e70da39aeba8c985242b Mon Sep 17 00:00:00 2001 From: filipe Date: Thu, 8 Oct 2020 20:14:54 -0300 Subject: [PATCH 139/441] fix pvs check --- src/Functions/FunctionsFormatting.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index b2de00516cc..18004af970b 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -455,9 +455,6 @@ private: const ColumnConst * maximum_unit_const_col = checkAndGetColumnConstStringOrFixedString(maximum_unit_column.get()); if (maximum_unit_const_col) maximum_unit = maximum_unit_const_col->getValue(); - else - throw Exception( - "Illegal column " + maximum_unit_const_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } if (const ColumnVector * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get())) @@ -484,7 +481,6 @@ private: block.getByPosition(result).column = std::move(col_to); return true; } - return false; } }; From fbd7141feda85f6fce3dc42ce403da63da1414c8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 9 Oct 2020 10:19:17 +0300 Subject: [PATCH 140/441] Fix flaky tests --- .../test_quorum_inserts_parallel/test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_quorum_inserts_parallel/test.py b/tests/integration/test_quorum_inserts_parallel/test.py index 57bb95b7113..c89f1a03df7 100644 --- a/tests/integration/test_quorum_inserts_parallel/test.py +++ b/tests/integration/test_quorum_inserts_parallel/test.py @@ -57,7 +57,6 @@ def test_parallel_quorum_actually_parallel(started_cluster): def test_parallel_quorum_actually_quorum(started_cluster): - settings = {"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "3000"} for i, node in enumerate([node1, node2, node3]): node.query("CREATE TABLE q (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/q', '{num}') ORDER BY tuple()".format(num=i)) @@ -65,7 +64,7 @@ def test_parallel_quorum_actually_quorum(started_cluster): pm.partition_instances(node2, node1, port=9009) pm.partition_instances(node2, node3, port=9009) with pytest.raises(QueryRuntimeException): - node1.query("INSERT INTO q VALUES(1, 'Hello')", settings=settings) + node1.query("INSERT INTO q VALUES(1, 'Hello')", settings={"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "3000"}) assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "1") assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "0") @@ -81,14 +80,21 @@ def test_parallel_quorum_actually_quorum(started_cluster): node.query("INSERT INTO q VALUES(3, 'Hi')", settings=settings) p = Pool(2) - res = p.apply_async(insert_value_to_node, (node1, {"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "10000"})) - insert_value_to_node(node2, {}) + res = p.apply_async(insert_value_to_node, (node1, {"insert_quorum": "3", "insert_quorum_parallel": "1", "insert_quorum_timeout": "60000"})) + + assert_eq_with_retry(node1, "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", "3") + assert_eq_with_retry(node3, "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", "3") + assert_eq_with_retry(node2, "SELECT COUNT() FROM system.parts WHERE table == 'q' and active == 1", "0") + + # Insert to the second to satisfy quorum + insert_value_to_node(node2, {"insert_quorum": "3", "insert_quorum_parallel": "1"}) + + res.get() assert_eq_with_retry(node1, "SELECT COUNT() FROM q", "3") assert_eq_with_retry(node2, "SELECT COUNT() FROM q", "1") assert_eq_with_retry(node3, "SELECT COUNT() FROM q", "3") - res.get() p.close() p.join() From a7fb2e38a5ac703abce313a0404eafb7da172d22 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 9 Oct 2020 10:41:28 +0300 Subject: [PATCH 141/441] Use ColumnWithTypeAndName as function argument instead of Block. --- .../AggregateFunctionMLMethod.cpp | 16 +-- .../AggregateFunctionMLMethod.h | 12 +- src/AggregateFunctions/IAggregateFunction.h | 2 +- src/Columns/ColumnAggregateFunction.cpp | 4 +- src/Columns/ColumnAggregateFunction.h | 2 +- src/Columns/ColumnFunction.cpp | 8 +- src/Core/Block.h | 6 + src/Core/DecimalComparison.h | 8 +- src/Functions/CustomWeekTransforms.h | 5 +- src/Functions/DateTimeTransforms.h | 5 +- src/Functions/FunctionBinaryArithmetic.h | 41 +++--- .../FunctionDateOrDateTimeAddInterval.h | 4 +- src/Functions/FunctionFactory.h | 1 + src/Functions/FunctionHelpers.cpp | 26 ++-- src/Functions/FunctionHelpers.h | 4 +- src/Functions/FunctionJoinGet.cpp | 6 +- src/Functions/FunctionJoinGet.h | 5 +- src/Functions/FunctionStartsEndsWith.h | 2 +- src/Functions/FunctionStringToString.h | 2 +- src/Functions/FunctionUnaryArithmetic.h | 2 +- src/Functions/FunctionsCoding.h | 20 +-- src/Functions/FunctionsComparison.h | 63 ++++----- src/Functions/FunctionsConversion.h | 64 +++++---- src/Functions/FunctionsEmbeddedDictionaries.h | 2 +- src/Functions/FunctionsExternalDictionaries.h | 10 +- src/Functions/FunctionsExternalModels.h | 1 + src/Functions/FunctionsFormatting.h | 2 +- src/Functions/FunctionsHashing.h | 4 +- src/Functions/FunctionsJSON.cpp | 2 +- src/Functions/FunctionsJSON.h | 36 ++--- src/Functions/FunctionsLogical.cpp | 2 +- src/Functions/FunctionsMiscellaneous.h | 2 +- src/Functions/FunctionsRandom.h | 2 +- src/Functions/FunctionsRound.h | 6 +- src/Functions/FunctionsStringArray.h | 8 +- src/Functions/IFunction.cpp | 130 ++++++++++-------- src/Functions/IFunction.h | 19 +-- src/Functions/IFunctionAdaptors.h | 29 ++-- src/Functions/IFunctionImpl.h | 56 +++++++- src/Functions/PerformanceAdaptors.h | 2 +- src/Functions/URL/URLHierarchy.cpp | 2 +- src/Functions/URL/URLPathHierarchy.cpp | 2 +- .../URL/extractURLParameterNames.cpp | 2 +- src/Functions/URL/extractURLParameters.cpp | 2 +- src/Functions/array/arrayElement.cpp | 22 +-- src/Functions/array/arrayIndex.h | 19 +-- src/Functions/array/arrayIntersect.cpp | 7 +- src/Functions/array/emptyArrayToSingle.cpp | 2 +- src/Functions/caseWithExpression.cpp | 20 +-- src/Functions/coalesce.cpp | 18 +-- src/Functions/concat.cpp | 2 +- src/Functions/dateDiff.cpp | 4 +- src/Functions/date_trunc.cpp | 14 +- src/Functions/defaultValueOfArgumentType.cpp | 4 +- src/Functions/evalMLMethod.cpp | 2 +- .../extractTimeZoneFromFunctionArguments.cpp | 8 +- .../extractTimeZoneFromFunctionArguments.h | 2 +- src/Functions/formatDateTime.cpp | 4 +- src/Functions/formatRow.cpp | 2 +- src/Functions/getSizeOfEnumType.cpp | 8 +- src/Functions/greatCircleDistance.cpp | 2 +- src/Functions/if.cpp | 42 +++--- src/Functions/ifNotFinite.cpp | 13 +- src/Functions/ifNull.cpp | 18 +-- src/Functions/ignore.cpp | 2 +- src/Functions/in.cpp | 2 +- src/Functions/nullIf.cpp | 16 +-- src/Functions/reverse.cpp | 2 +- src/Functions/toColumnTypeName.cpp | 4 +- src/Functions/toFixedString.h | 2 +- src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toTypeName.cpp | 2 +- src/Functions/transform.cpp | 14 +- src/Functions/tuple.cpp | 2 +- src/Interpreters/ArrayJoinAction.cpp | 18 +-- src/Interpreters/DictionaryReader.cpp | 10 +- src/Interpreters/DictionaryReader.h | 4 +- src/Interpreters/ExpressionActions.cpp | 14 +- src/Interpreters/ExpressionJIT.cpp | 8 +- src/Interpreters/ExpressionJIT.h | 4 +- .../RemoveInjectiveFunctionsVisitor.cpp | 2 +- src/Interpreters/TreeOptimizer.cpp | 2 +- src/Interpreters/castColumn.cpp | 13 +- src/Storages/MergeTree/KeyCondition.cpp | 38 ++--- src/Storages/MergeTree/KeyCondition.h | 10 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 8 +- 86 files changed, 556 insertions(+), 465 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index 413209fd2dc..5cbdf4c3b45 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -143,13 +143,13 @@ void LinearModelData::updateState() void LinearModelData::predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, const Context & context) const { - gradient_computer->predict(container, block, offset, limit, arguments, weights, bias, context); + gradient_computer->predict(container, columns, offset, limit, arguments, weights, bias, context); } void LinearModelData::returnWeights(IColumn & to) const @@ -449,7 +449,7 @@ void IWeightsUpdater::addToBatch( void LogisticRegression::predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -457,7 +457,7 @@ void LogisticRegression::predict( Float64 bias, const Context & /*context*/) const { - size_t rows_num = block.rows(); + size_t rows_num = columns[arguments.front()].column->size(); if (offset > rows_num || offset + limit > rows_num) throw Exception("Invalid offset and limit for LogisticRegression::predict. " @@ -468,7 +468,7 @@ void LogisticRegression::predict( for (size_t i = 1; i < arguments.size(); ++i) { - const ColumnWithTypeAndName & cur_col = block.getByPosition(arguments[i]); + const ColumnWithTypeAndName & cur_col = columns[arguments[i]]; if (!isNativeNumber(cur_col.type)) throw Exception("Prediction arguments must have numeric type", ErrorCodes::BAD_ARGUMENTS); @@ -518,7 +518,7 @@ void LogisticRegression::compute( void LinearRegression::predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -531,7 +531,7 @@ void LinearRegression::predict( throw Exception("In predict function number of arguments differs from the size of weights vector", ErrorCodes::LOGICAL_ERROR); } - size_t rows_num = block.rows(); + size_t rows_num = columns[arguments.front()].column->size(); if (offset > rows_num || offset + limit > rows_num) throw Exception("Invalid offset and limit for LogisticRegression::predict. " @@ -542,7 +542,7 @@ void LinearRegression::predict( for (size_t i = 1; i < arguments.size(); ++i) { - const ColumnWithTypeAndName & cur_col = block.getByPosition(arguments[i]); + const ColumnWithTypeAndName & cur_col = columns[arguments[i]]; if (!isNativeNumber(cur_col.type)) throw Exception("Prediction arguments must have numeric type", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.h b/src/AggregateFunctions/AggregateFunctionMLMethod.h index 8a93b66ab3b..6f7c1232f4a 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -39,7 +39,7 @@ public: virtual void predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -65,7 +65,7 @@ public: void predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -91,7 +91,7 @@ public: void predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -264,7 +264,7 @@ public: void predict( ColumnVector::Container & container, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -364,7 +364,7 @@ public: void predictValues( ConstAggregateDataPtr place, IColumn & to, - Block & block, + ColumnsWithTypeAndName & columns, size_t offset, size_t limit, const ColumnNumbers & arguments, @@ -382,7 +382,7 @@ public: throw Exception("Cast of column of predictions is incorrect. getReturnTypeToPredict must return same value as it is casted to", ErrorCodes::LOGICAL_ERROR); - this->data(place).predict(column->getData(), block, offset, limit, arguments, context); + this->data(place).predict(column->getData(), columns, offset, limit, arguments, context); } /** This function is called if aggregate function without State modifier is selected in a query. diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index b9656c31fa3..498a5303ac4 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -114,7 +114,7 @@ public: virtual void predictValues( ConstAggregateDataPtr /* place */, IColumn & /*to*/, - Block & /*block*/, + ColumnsWithTypeAndName & /*block*/, size_t /*offset*/, size_t /*limit*/, const ColumnNumbers & /*arguments*/, diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index b9200ffe910..189ca12b5f7 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -161,7 +161,7 @@ MutableColumnPtr ColumnAggregateFunction::convertToValues(MutableColumnPtr colum return res; } -MutableColumnPtr ColumnAggregateFunction::predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const +MutableColumnPtr ColumnAggregateFunction::predictValues(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const Context & context) const { MutableColumnPtr res = func->getReturnTypeToPredict()->createColumn(); res->reserve(data.size()); @@ -172,7 +172,7 @@ MutableColumnPtr ColumnAggregateFunction::predictValues(Block & block, const Col if (data.size() == 1) { /// Case for const column. Predict using single model. - machine_learning_function->predictValues(data[0], *res, block, 0, block.rows(), arguments, context); + machine_learning_function->predictValues(data[0], *res, block, 0, block[arguments.front()].column->size(), arguments, context); } else { diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 015fc0fd789..19b9bd09a7a 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -119,7 +119,7 @@ public: const char * getFamilyName() const override { return "AggregateFunction"; } TypeIndex getDataType() const override { return TypeIndex::AggregateFunction; } - MutableColumnPtr predictValues(Block & block, const ColumnNumbers & arguments, const Context & context) const; + MutableColumnPtr predictValues(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const Context & context) const; size_t size() const override { diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp index 24c074187a4..25d1a75b0e2 100644 --- a/src/Columns/ColumnFunction.cpp +++ b/src/Columns/ColumnFunction.cpp @@ -187,16 +187,16 @@ ColumnWithTypeAndName ColumnFunction::reduce() const throw Exception("Cannot call function " + function->getName() + " because is has " + toString(args) + "arguments but " + toString(captured) + " columns were captured.", ErrorCodes::LOGICAL_ERROR); - Block block(captured_columns); - block.insert({nullptr, function->getReturnType(), ""}); + auto columns = captured_columns; + columns.emplace_back(ColumnWithTypeAndName {nullptr, function->getReturnType(), ""}); ColumnNumbers arguments(captured_columns.size()); for (size_t i = 0; i < captured_columns.size(); ++i) arguments[i] = i; - function->execute(block, arguments, captured_columns.size(), size_); + function->execute(columns, arguments, captured_columns.size(), size_); - return block.getByPosition(captured_columns.size()); + return columns[captured_columns.size()]; } } diff --git a/src/Core/Block.h b/src/Core/Block.h index 31e3ffc14de..f05cc2b52eb 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -152,6 +152,12 @@ public: private: void eraseImpl(size_t position); void initializeIndexByName(); + + /// This is needed to allow function execution over data. + /// It is safe because functions does not change column names, so index is unaffected. + /// It is temporary. + friend struct ExpressionAction; + friend class ActionsDAG; }; using Blocks = std::vector; diff --git a/src/Core/DecimalComparison.h b/src/Core/DecimalComparison.h index b9ae2a1fe79..8c88afb5be9 100644 --- a/src/Core/DecimalComparison.h +++ b/src/Core/DecimalComparison.h @@ -60,14 +60,14 @@ public: using ArrayA = typename ColVecA::Container; using ArrayB = typename ColVecB::Container; - DecimalComparison(Block & block, size_t result, const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) + DecimalComparison(ColumnsWithTypeAndName & data, size_t result, const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) { - if (!apply(block, result, col_left, col_right)) + if (!apply(data, result, col_left, col_right)) throw Exception("Wrong decimal comparison with " + col_left.type->getName() + " and " + col_right.type->getName(), ErrorCodes::LOGICAL_ERROR); } - static bool apply(Block & block, size_t result [[maybe_unused]], + static bool apply(ColumnsWithTypeAndName & data, size_t result [[maybe_unused]], const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) { if constexpr (_actual) @@ -77,7 +77,7 @@ public: c_res = applyWithScale(col_left.column, col_right.column, shift); if (c_res) - block.getByPosition(result).column = std::move(c_res); + data[result].column = std::move(c_res); return true; } return false; diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index 86e1c444a78..4a9f2d5f556 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -115,7 +116,7 @@ template struct CustomWeekTransformImpl { template - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, Transform transform = {}) + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, Transform transform = {}) { const auto op = Transformer{std::move(transform)}; @@ -126,7 +127,7 @@ struct CustomWeekTransformImpl week_mode = week_mode_column->getValue(); } - const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0); + const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block.data, arguments, 2, 0); const ColumnPtr source_col = block.getByPosition(arguments[0]).column; if (const auto * sources = checkAndGetColumn(source_col.get())) { diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 6220d10a17d..6597cc281a4 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -682,11 +683,11 @@ struct Transformer template struct DateTimeTransformImpl { - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, const Transform & transform = {}) + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, const Transform & transform = {}) { using Op = Transformer; - const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 1, 0); + const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block.data, arguments, 1, 0); const ColumnPtr source_col = block.getByPosition(arguments[0]).column; if (const auto * sources = checkAndGetColumn(source_col.get())) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index bbb08c4068f..a357c41c636 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -729,7 +729,7 @@ class FunctionBinaryArithmetic : public IFunction {new_block.getByPosition(new_arguments[0]), new_block.getByPosition(new_arguments[1])}; auto function = function_builder->build(new_arguments_with_type_and_name); - function->execute(new_block, new_arguments, result, input_rows_count); + function->execute(new_block.data, new_arguments, result, input_rows_count); block.getByPosition(result).column = new_block.getByPosition(result).column; } @@ -872,7 +872,7 @@ public: col_right->getChars().data(), out_chars.data(), out_chars.size()); - block.getByPosition(result).column = ColumnConst::create(std::move(col_res), block.rows()); + block.getByPosition(result).column = ColumnConst::create(std::move(col_res), col_left_raw->size()); return true; } } @@ -988,7 +988,7 @@ public: col_res = ColVecResult::create(0, type.getScale()); auto & vec_res = col_res->getData(); - vec_res.resize(block.rows()); + vec_res.resize(col_left_raw->size()); if (col_left && col_right) { @@ -1032,7 +1032,7 @@ public: col_res = ColVecResult::create(); auto & vec_res = col_res->getData(); - vec_res.resize(block.rows()); + vec_res.resize(col_left_raw->size()); if (col_left && col_right) { @@ -1171,6 +1171,7 @@ class FunctionBinaryArithmeticWithConstants : public FunctionBinaryArithmetic; using Monotonicity = typename Base::Monotonicity; + using Block = typename Base::Block; static FunctionPtr create( const ColumnWithTypeAndName & left_, @@ -1194,21 +1195,25 @@ public: { if (left.column && isColumnConst(*left.column) && arguments.size() == 1) { - Block block_with_constant + ColumnsWithTypeAndName block_with_constant = {{left.column->cloneResized(input_rows_count), left.type, left.name}, block.getByPosition(arguments[0]), block.getByPosition(result)}; - Base::executeImpl(block_with_constant, {0, 1}, 2, input_rows_count); - block.getByPosition(result) = block_with_constant.getByPosition(2); + + FunctionArguments args(block_with_constant); + Base::executeImpl(args, {0, 1}, 2, input_rows_count); + block.getByPosition(result) = block_with_constant[2]; } else if (right.column && isColumnConst(*right.column) && arguments.size() == 1) { - Block block_with_constant + ColumnsWithTypeAndName block_with_constant = {block.getByPosition(arguments[0]), {right.column->cloneResized(input_rows_count), right.type, right.name}, block.getByPosition(result)}; - Base::executeImpl(block_with_constant, {0, 1}, 2, input_rows_count); - block.getByPosition(result) = block_with_constant.getByPosition(2); + + FunctionArguments args(block_with_constant); + Base::executeImpl(args, {0, 1}, 2, input_rows_count); + block.getByPosition(result) = block_with_constant[2]; } else Base::executeImpl(block, arguments, result, input_rows_count); @@ -1242,13 +1247,15 @@ public: { auto transform = [&](const Field & point) { - Block block_with_constant + ColumnsWithTypeAndName block_with_constant = {{left.column->cloneResized(1), left.type, left.name}, {right.type->createColumnConst(1, point), right.type, right.name}, {nullptr, return_type, ""}}; - Base::executeImpl(block_with_constant, {0, 1}, 2, 1); + FunctionArguments args(block_with_constant); + + Base::executeImpl(args, {0, 1}, 2, 1); Field point_transformed; - block_with_constant.getByPosition(2).column->get(0, point_transformed); + block_with_constant[2].column->get(0, point_transformed); return point_transformed; }; transform(left_point); @@ -1277,13 +1284,15 @@ public: { auto transform = [&](const Field & point) { - Block block_with_constant + ColumnsWithTypeAndName block_with_constant = {{left.type->createColumnConst(1, point), left.type, left.name}, {right.column->cloneResized(1), right.type, right.name}, {nullptr, return_type, ""}}; - Base::executeImpl(block_with_constant, {0, 1}, 2, 1); + + FunctionArguments args(block_with_constant); + Base::executeImpl(args, {0, 1}, 2, 1); Field point_transformed; - block_with_constant.getByPosition(2).column->get(0, point_transformed); + block_with_constant[2].column->get(0, point_transformed); return point_transformed; }; diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 92a6efe9fb7..c2d171fc2f6 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -304,7 +304,7 @@ private: template struct DateTimeAddIntervalImpl { - static void execute(Transform transform, Block & block, const ColumnNumbers & arguments, size_t result) + static void execute(Transform transform, FunctionArguments & block, const ColumnNumbers & arguments, size_t result) { using FromValueType = typename FromDataType::FieldType; using FromColumnType = typename FromDataType::ColumnType; @@ -312,7 +312,7 @@ struct DateTimeAddIntervalImpl auto op = Adder{std::move(transform)}; - const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block, arguments, 2, 0); + const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(block.data, arguments, 2, 0); const ColumnPtr source_col = block.getByPosition(arguments[0]).column; diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index 7990e78daf8..7872c192b41 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 18e5fde5462..473feeaca60 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -51,14 +51,14 @@ Columns convertConstTupleToConstantElements(const ColumnConst & column) } -static Block createBlockWithNestedColumnsImpl(const Block & block, const std::unordered_set & args) +static ColumnsWithTypeAndName createBlockWithNestedColumnsImpl(const ColumnsWithTypeAndName & columns, const std::unordered_set & args) { - Block res; - size_t columns = block.columns(); + ColumnsWithTypeAndName res; + size_t num_columns = columns.size(); - for (size_t i = 0; i < columns; ++i) + for (size_t i = 0; i < num_columns; ++i) { - const auto & col = block.getByPosition(i); + const auto & col = columns[i]; if (args.count(i) && col.type->isNullable()) { @@ -66,40 +66,40 @@ static Block createBlockWithNestedColumnsImpl(const Block & block, const std::un if (!col.column) { - res.insert({nullptr, nested_type, col.name}); + res.emplace_back(ColumnWithTypeAndName{nullptr, nested_type, col.name}); } else if (const auto * nullable = checkAndGetColumn(*col.column)) { const auto & nested_col = nullable->getNestedColumnPtr(); - res.insert({nested_col, nested_type, col.name}); + res.emplace_back(ColumnWithTypeAndName{nested_col, nested_type, col.name}); } else if (const auto * const_column = checkAndGetColumn(*col.column)) { const auto & nested_col = checkAndGetColumn(const_column->getDataColumn())->getNestedColumnPtr(); - res.insert({ ColumnConst::create(nested_col, col.column->size()), nested_type, col.name}); + res.emplace_back(ColumnWithTypeAndName{ ColumnConst::create(nested_col, col.column->size()), nested_type, col.name}); } else throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN); } else - res.insert(col); + res.emplace_back(col); } return res; } -Block createBlockWithNestedColumns(const Block & block, const ColumnNumbers & args) +ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args) { std::unordered_set args_set(args.begin(), args.end()); - return createBlockWithNestedColumnsImpl(block, args_set); + return createBlockWithNestedColumnsImpl(columns, args_set); } -Block createBlockWithNestedColumns(const Block & block, const ColumnNumbers & args, size_t result) +ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args, size_t result) { std::unordered_set args_set(args.begin(), args.end()); args_set.insert(result); - return createBlockWithNestedColumnsImpl(block, args_set); + return createBlockWithNestedColumnsImpl(columns, args_set); } void validateArgumentType(const IFunction & func, const DataTypes & arguments, diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 34aa0add6e1..02ef1804d8d 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -85,10 +85,10 @@ Columns convertConstTupleToConstantElements(const ColumnConst & column); /// Returns the copy of a given block in which each column specified in /// the "arguments" parameter is replaced with its respective nested /// column if it is nullable. -Block createBlockWithNestedColumns(const Block & block, const ColumnNumbers & args); +ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args); /// Similar function as above. Additionally transform the result type if needed. -Block createBlockWithNestedColumns(const Block & block, const ColumnNumbers & args, size_t result); +ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns, const ColumnNumbers & args, size_t result); /// Checks argument type at specified index with predicate. /// throws if there is no argument at specified index or if predicate returns false. diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index 1badc689c6a..de66a593ce8 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -19,11 +19,11 @@ namespace ErrorCodes template void ExecutableFunctionJoinGet::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t) { - Block keys; + ColumnsWithTypeAndName keys; for (size_t i = 2; i < arguments.size(); ++i) { auto key = block.getByPosition(arguments[i]); - keys.insert(std::move(key)); + keys.emplace_back(std::move(key)); } block.getByPosition(result) = join->joinGet(keys, result_block); } @@ -31,7 +31,7 @@ void ExecutableFunctionJoinGet::execute(Block & block, const ColumnNumb template ExecutableFunctionImplPtr FunctionJoinGet::prepare(const Block &, const ColumnNumbers &, size_t) const { - return std::make_unique>(join, Block{{return_type->createColumn(), return_type, attr_name}}); + return std::make_unique>(join, DB::Block{{return_type->createColumn(), return_type, attr_name}}); } static auto getJoin(const ColumnsWithTypeAndName & arguments, const Context & context) diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index e1afd2715f0..524d07ad0a6 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB { @@ -13,7 +14,7 @@ template class ExecutableFunctionJoinGet final : public IExecutableFunctionImpl { public: - ExecutableFunctionJoinGet(HashJoinPtr join_, const Block & result_block_) + ExecutableFunctionJoinGet(HashJoinPtr join_, const DB::Block & result_block_) : join(std::move(join_)), result_block(result_block_) {} static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; @@ -28,7 +29,7 @@ public: private: HashJoinPtr join; - Block result_block; + DB::Block result_block; }; template diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index 1587fa21cec..e4ee748427b 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -158,7 +158,7 @@ public: #endif } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { selector.selectAndExecute(block, arguments, result, input_rows_count); } diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h index 827a81af158..1e6aa9516be 100644 --- a/src/Functions/FunctionStringToString.h +++ b/src/Functions/FunctionStringToString.h @@ -35,7 +35,7 @@ public: return 1; } - bool isInjective(const Block &) const override + bool isInjective(const ColumnsWithTypeAndName &) const override { return is_injective; } diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 94f3fd291de..500222e4f03 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -117,7 +117,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return is_injective; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return is_injective; } bool useDefaultImplementationForConstants() const override { return true; } diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 8b7dc616330..fee10f1365a 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -72,7 +72,7 @@ public: String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -326,7 +326,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return mask_tail_octets == 0; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return mask_tail_octets == 0; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -447,7 +447,7 @@ public: String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -546,7 +546,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -739,7 +739,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -837,7 +837,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -941,7 +941,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -1224,7 +1224,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -1313,7 +1313,7 @@ public: } bool isVariadic() const override { return true; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } size_t getNumberOfArguments() const override { return 0; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override @@ -1408,7 +1408,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 436502aead4..75b8fdd51a2 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -688,9 +688,9 @@ private: using RightDataType = typename Types::RightType; if (check_decimal_overflow) - DecimalComparison(block, result, col_left, col_right); + DecimalComparison(block.data, result, col_left, col_right); else - DecimalComparison(block, result, col_left, col_right); + DecimalComparison(block.data, result, col_left, col_right); return true; }; @@ -852,13 +852,14 @@ private: { auto column_converted = type_to_compare->createColumnConst(input_rows_count, converted); - Block tmp_block + ColumnsWithTypeAndName tmp_block_columns { { left_const ? column_converted : col_left_untyped->getPtr(), type_to_compare, "" }, { !left_const ? column_converted : col_right_untyped->getPtr(), type_to_compare, "" }, block.getByPosition(result) }; + FunctionArguments tmp_block(tmp_block_columns); executeImpl(tmp_block, {0, 1}, 2, input_rows_count); block.getByPosition(result).column = std::move(tmp_block.getByPosition(2).column); @@ -949,24 +950,24 @@ private: ColumnsWithTypeAndName convolution_types(tuple_size); - Block tmp_block; + ColumnsWithTypeAndName tmp_block; for (size_t i = 0; i < tuple_size; ++i) { - tmp_block.insert(x[i]); - tmp_block.insert(y[i]); + tmp_block.emplace_back(x[i]); + tmp_block.emplace_back(y[i]); auto impl = func_compare->build({x[i], y[i]}); convolution_types[i].type = impl->getReturnType(); /// Comparison of the elements. - tmp_block.insert({ nullptr, impl->getReturnType(), "" }); + tmp_block.emplace_back(ColumnWithTypeAndName{ nullptr, impl->getReturnType(), "" }); impl->execute(tmp_block, {i * 3, i * 3 + 1}, i * 3 + 2, input_rows_count); } if (tuple_size == 1) { /// Do not call AND for single-element tuple. - block.getByPosition(result).column = tmp_block.getByPosition(2).column; + block.getByPosition(result).column = tmp_block[2].column; return; } @@ -977,10 +978,10 @@ private: convolution_args[i] = i * 3 + 2; auto impl = func_convolution->build(convolution_types); - tmp_block.insert({ nullptr, impl->getReturnType(), "" }); + tmp_block.emplace_back(ColumnWithTypeAndName{ nullptr, impl->getReturnType(), "" }); impl->execute(tmp_block, convolution_args, tuple_size * 3, input_rows_count); - block.getByPosition(result).column = tmp_block.getByPosition(tuple_size * 3).column; + block.getByPosition(result).column = tmp_block[tuple_size * 3].column; } void executeTupleLessGreaterImpl( @@ -996,34 +997,34 @@ private: size_t tuple_size, size_t input_rows_count) const { - Block tmp_block; + ColumnsWithTypeAndName tmp_block; /// Pairwise comparison of the inequality of all elements; on the equality of all elements except the last. /// (x[i], y[i], x[i] < y[i], x[i] == y[i]) for (size_t i = 0; i < tuple_size; ++i) { - tmp_block.insert(x[i]); - tmp_block.insert(y[i]); + tmp_block.emplace_back(x[i]); + tmp_block.emplace_back(y[i]); - tmp_block.insert(ColumnWithTypeAndName()); // pos == i * 4 + 2 + tmp_block.emplace_back(ColumnWithTypeAndName()); // pos == i * 4 + 2 if (i + 1 != tuple_size) { auto impl_head = func_compare_head->build({x[i], y[i]}); - tmp_block.getByPosition(i * 4 + 2).type = impl_head->getReturnType(); + tmp_block[i * 4 + 2].type = impl_head->getReturnType(); impl_head->execute(tmp_block, {i * 4, i * 4 + 1}, i * 4 + 2, input_rows_count); - tmp_block.insert(ColumnWithTypeAndName()); // i * 4 + 3 + tmp_block.emplace_back(ColumnWithTypeAndName()); // i * 4 + 3 auto impl_equals = func_equals->build({x[i], y[i]}); - tmp_block.getByPosition(i * 4 + 3).type = impl_equals->getReturnType(); + tmp_block[i * 4 + 3].type = impl_equals->getReturnType(); impl_equals->execute(tmp_block, {i * 4, i * 4 + 1}, i * 4 + 3, input_rows_count); } else { auto impl_tail = func_compare_tail->build({x[i], y[i]}); - tmp_block.getByPosition(i * 4 + 2).type = impl_tail->getReturnType(); + tmp_block[i * 4 + 2].type = impl_tail->getReturnType(); impl_tail->execute(tmp_block, {i * 4, i * 4 + 1}, i * 4 + 2, input_rows_count); } } @@ -1039,31 +1040,31 @@ private: { --i; - size_t and_lhs_pos = tmp_block.columns() - 1; // res + size_t and_lhs_pos = tmp_block.size() - 1; // res size_t and_rhs_pos = i * 4 + 3; // `x == y`[i] - tmp_block.insert(ColumnWithTypeAndName()); + tmp_block.emplace_back(ColumnWithTypeAndName()); - ColumnsWithTypeAndName and_args = {{ nullptr, tmp_block.getByPosition(and_lhs_pos).type, "" }, - { nullptr, tmp_block.getByPosition(and_rhs_pos).type, "" }}; + ColumnsWithTypeAndName and_args = {{ nullptr, tmp_block[and_lhs_pos].type, "" }, + { nullptr, tmp_block[and_rhs_pos].type, "" }}; auto func_and_adaptor = func_and->build(and_args); - tmp_block.getByPosition(tmp_block.columns() - 1).type = func_and_adaptor->getReturnType(); - func_and_adaptor->execute(tmp_block, {and_lhs_pos, and_rhs_pos}, tmp_block.columns() - 1, input_rows_count); + tmp_block[tmp_block.size() - 1].type = func_and_adaptor->getReturnType(); + func_and_adaptor->execute(tmp_block, {and_lhs_pos, and_rhs_pos}, tmp_block.size() - 1, input_rows_count); - size_t or_lhs_pos = tmp_block.columns() - 1; // (res && `x == y`[i]) + size_t or_lhs_pos = tmp_block.size() - 1; // (res && `x == y`[i]) size_t or_rhs_pos = i * 4 + 2; // `x < y`[i] - tmp_block.insert(ColumnWithTypeAndName()); + tmp_block.emplace_back(ColumnWithTypeAndName()); - ColumnsWithTypeAndName or_args = {{ nullptr, tmp_block.getByPosition(or_lhs_pos).type, "" }, - { nullptr, tmp_block.getByPosition(or_rhs_pos).type, "" }}; + ColumnsWithTypeAndName or_args = {{ nullptr, tmp_block[or_lhs_pos].type, "" }, + { nullptr, tmp_block[or_rhs_pos].type, "" }}; auto func_or_adaptor = func_or->build(or_args); - tmp_block.getByPosition(tmp_block.columns() - 1).type = func_or_adaptor->getReturnType(); - func_or_adaptor->execute(tmp_block, {or_lhs_pos, or_rhs_pos}, tmp_block.columns() - 1, input_rows_count); + tmp_block[tmp_block.size() - 1].type = func_or_adaptor->getReturnType(); + func_or_adaptor->execute(tmp_block, {or_lhs_pos, or_rhs_pos}, tmp_block.size() - 1, input_rows_count); } - block.getByPosition(result).column = tmp_block.getByPosition(tmp_block.columns() - 1).column; + block.getByPosition(result).column = tmp_block[tmp_block.size() - 1].column; } void executeGenericIdenticalTypes(Block & block, size_t result, const IColumn * c0, const IColumn * c1) const diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 2210c61d157..570641c3833 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -101,7 +101,7 @@ struct ConvertImpl using ToFieldType = typename ToDataType::FieldType; template - static void NO_SANITIZE_UNDEFINED execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, + static void NO_SANITIZE_UNDEFINED execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/, Additions additions [[maybe_unused]] = Additions()) { const ColumnWithTypeAndName & named_from = block.getByPosition(arguments[0]); @@ -441,7 +441,7 @@ struct FormatImpl> template struct ConvertImpl, DataTypeNumber, Name> { - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { block.getByPosition(result).column = block.getByPosition(arguments[0]).column; } @@ -454,7 +454,7 @@ struct ConvertImpl, ColumnDecimal, ColumnVector>; - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { const auto & col_with_type_and_name = block.getByPosition(arguments[0]); const auto & type = static_cast(*col_with_type_and_name.type); @@ -463,7 +463,7 @@ struct ConvertImpl || std::is_same_v) - time_zone = &extractTimeZoneFromFunctionArguments(block, arguments, 1, 0); + time_zone = &extractTimeZoneFromFunctionArguments(block.data, arguments, 1, 0); if (const auto col_from = checkAndGetColumn(col_with_type_and_name.column.get())) { @@ -508,7 +508,7 @@ struct ConvertImpl(DataTypeDateTime::FieldType & x, Read /** Throw exception with verbose message when string value is not parsed completely. */ -[[noreturn]] inline void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result) +[[noreturn]] inline void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, FunctionArguments & block, size_t result) { const IDataType & to_type = *block.getByPosition(result).type; @@ -670,7 +670,7 @@ struct ConvertThroughParsing } template - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count, + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count, Additions additions [[maybe_unused]] = Additions()) { using ColVecTo = typename ToDataType::ColumnType; @@ -687,7 +687,7 @@ struct ConvertThroughParsing local_time_zone = &dt_col->getTimeZone(); else { - local_time_zone = &extractTimeZoneFromFunctionArguments(block, arguments, 1, 0); + local_time_zone = &extractTimeZoneFromFunctionArguments(block.data, arguments, 1, 0); } if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffort || parsing_mode == ConvertFromStringParsingMode::BestEffortUS) @@ -865,7 +865,7 @@ struct ConvertImpl template struct ConvertImpl, T, Name> { - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { block.getByPosition(result).column = block.getByPosition(arguments[0]).column; } @@ -933,7 +933,7 @@ struct ConvertImpl, T, Name> template struct ConvertImpl { - static void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) + static void execute(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) { if (const ColumnFixedString * col_from = checkAndGetColumn(block.getByPosition(arguments[0]).column.get())) { @@ -1048,7 +1048,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } - bool isInjective(const Block &) const override { return std::is_same_v; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return std::is_same_v; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -1952,7 +1952,7 @@ private: return [function_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { - function_adaptor->execute(block, arguments, result, input_rows_count); + function_adaptor->execute(block.data, arguments, result, input_rows_count); }; } @@ -1966,7 +1966,7 @@ private: return [function_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { - function_adaptor->execute(block, arguments, result, input_rows_count); + function_adaptor->execute(block.data, arguments, result, input_rows_count); }; } @@ -1994,7 +1994,7 @@ private: return [function_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { - function_adaptor->execute(block, arguments, result, input_rows_count); + function_adaptor->execute(block.data, arguments, result, input_rows_count); }; } @@ -2093,14 +2093,15 @@ private: if (const ColumnArray * col_array = checkAndGetColumn(array_arg.column.get())) { /// create block for converting nested column containing original and result columns - Block nested_block + ColumnsWithTypeAndName nested_block_columns { { col_array->getDataPtr(), from_nested_type, "" }, { nullptr, to_nested_type, "" } }; + Block nested_block(nested_block_columns); /// convert nested column - nested_function(nested_block, {0}, 1, nested_block.rows()); + nested_function(nested_block, {0}, 1, nested_block_columns.front().column->size()); /// set converted nested column to result block.getByPosition(result).column = ColumnArray::create(nested_block.getByPosition(1).column, col_array->getOffsetsPtr()); @@ -2145,21 +2146,23 @@ private: const auto col = block.getByPosition(arguments.front()).column.get(); /// copy tuple elements to a separate block - Block element_block; + ColumnsWithTypeAndName element_block_columns; size_t tuple_size = from_element_types.size(); const ColumnTuple & column_tuple = typeid_cast(*col); /// create columns for source elements for (size_t i = 0; i < tuple_size; ++i) - element_block.insert({ column_tuple.getColumns()[i], from_element_types[i], "" }); + element_block_columns.emplace_back(ColumnWithTypeAndName{ column_tuple.getColumns()[i], from_element_types[i], "" }); /// create columns for converted elements for (const auto & to_element_type : to_element_types) - element_block.insert({ nullptr, to_element_type, "" }); + element_block_columns.emplace_back(ColumnWithTypeAndName{ nullptr, to_element_type, "" }); /// insert column for converted tuple - element_block.insert({ nullptr, std::make_shared(to_element_types), "" }); + element_block_columns.emplace_back(ColumnWithTypeAndName{ nullptr, std::make_shared(to_element_types), "" }); + + FunctionArguments element_block(element_block_columns); /// invoke conversion for each element for (const auto idx_element_wrapper : ext::enumerate(element_wrappers)) @@ -2197,7 +2200,7 @@ private: return [func_or_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { - func_or_adaptor->execute(block, arguments, result, input_rows_count); + func_or_adaptor->execute(block.data, arguments, result, input_rows_count); }; } else @@ -2422,22 +2425,24 @@ private: const auto & nullable_type = static_cast(*ret_type); const auto & nested_type = nullable_type.getNestedType(); - Block tmp_block; + ColumnsWithTypeAndName tmp_block_columns; if (source_is_nullable) - tmp_block = createBlockWithNestedColumns(block, arguments); + tmp_block_columns = createBlockWithNestedColumns(block.data, arguments); else - tmp_block = block; + tmp_block_columns = block.data; size_t tmp_res_index = block.columns(); - tmp_block.insert({nullptr, nested_type, ""}); + tmp_block_columns.emplace_back(ColumnWithTypeAndName {nullptr, nested_type, ""}); /// Add original ColumnNullable for createStringToEnumWrapper() if (source_is_nullable) { if (arguments.size() != 1) throw Exception("Invalid number of arguments", ErrorCodes::LOGICAL_ERROR); - tmp_block.insert(block.getByPosition(arguments.front())); + tmp_block_columns.emplace_back(block.getByPosition(arguments.front())); } + FunctionArguments tmp_block(tmp_block_columns); + /// Perform the requested conversion. wrapper(tmp_block, arguments, tmp_res_index, input_rows_count); @@ -2448,7 +2453,7 @@ private: throw Exception("Couldn't convert " + block.getByPosition(arguments[0]).type->getName() + " to " + nested_type->getName() + " in " + " prepareRemoveNullable wrapper.", ErrorCodes::LOGICAL_ERROR); - res.column = wrapInNullable(tmp_res.column, Block({block.getByPosition(arguments[0]), tmp_res}), {0}, 1, input_rows_count); + res.column = wrapInNullable(tmp_res.column, {block.getByPosition(arguments[0]), tmp_res}, {0}, 1, input_rows_count); }; } else if (source_is_nullable) @@ -2457,7 +2462,7 @@ private: return [wrapper, skip_not_null_check] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { - Block tmp_block = createBlockWithNestedColumns(block, arguments, result); + auto tmp_block_columns = createBlockWithNestedColumns(block.data, arguments, result); /// Check that all values are not-NULL. /// Check can be skipped in case if LowCardinality dictionary is transformed. @@ -2473,6 +2478,7 @@ private: ErrorCodes::CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN}; } + FunctionArguments tmp_block(tmp_block_columns); wrapper(tmp_block, arguments, result, input_rows_count); block.getByPosition(result).column = tmp_block.getByPosition(result).column; }; diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index 85d142a44ae..b00f6a4ced9 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -593,7 +593,7 @@ public: /// For the purpose of query optimization, we assume this function to be injective /// even in face of fact that there are many different cities named Moscow. - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 609c247ce42..63764afe6d2 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -283,7 +283,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) const override + bool isInjective(const ColumnsWithTypeAndName & sample_block) const override { return helper.isDictGetFunctionInjective(sample_block); } @@ -813,7 +813,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) const override + bool isInjective(const ColumnsWithTypeAndName & sample_block) const override { return helper.isDictGetFunctionInjective(sample_block); } @@ -1385,7 +1385,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) const override + bool isInjective(const ColumnsWithTypeAndName & sample_block) const override { return helper.isDictGetFunctionInjective(sample_block); } @@ -1533,7 +1533,7 @@ private: bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0, 1}; } - bool isInjective(const Block & sample_block) const override + bool isInjective(const ColumnsWithTypeAndName & sample_block) const override { return helper.isDictGetFunctionInjective(sample_block); } @@ -1672,7 +1672,7 @@ public: private: size_t getNumberOfArguments() const override { return 2; } - bool isInjective(const Block & /*sample_block*/) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName & /*sample_block*/) const override { return true; } bool useDefaultImplementationForConstants() const final { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } diff --git a/src/Functions/FunctionsExternalModels.h b/src/Functions/FunctionsExternalModels.h index 7cbe703d106..5c26dd36d85 100644 --- a/src/Functions/FunctionsExternalModels.h +++ b/src/Functions/FunctionsExternalModels.h @@ -5,6 +5,7 @@ namespace DB { class ExternalModelsLoader; +class Context; /// Evaluate external model. /// First argument - model name, the others - model arguments. diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index da22babec38..afadc27fe0d 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -42,7 +42,7 @@ public: } size_t getNumberOfArguments() const override { return 1; } - bool isInjective(const Block &) const override { return true; } + bool isInjective(const ColumnsWithTypeAndName &) const override { return true; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index 4a742ff5780..d36310721c2 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -689,7 +689,7 @@ public: #endif } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { selector.selectAndExecute(block, arguments, result, input_rows_count); } @@ -1086,7 +1086,7 @@ public: #endif } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(FunctionArguments & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { selector.selectAndExecute(block, arguments, result, input_rows_count); } diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index e3681f028d1..ae4c187b01d 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -10,7 +10,7 @@ namespace ErrorCodes } -std::vector FunctionJSONHelpers::prepareMoves(const char * function_name, Block & block, const ColumnNumbers & arguments, size_t first_index_argument, size_t num_index_arguments) +std::vector FunctionJSONHelpers::prepareMoves(const char * function_name, FunctionArguments & block, const ColumnNumbers & arguments, size_t first_index_argument, size_t num_index_arguments) { std::vector moves; moves.reserve(num_index_arguments); diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index a5c88cad1d6..9d9f06bf1e5 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -55,7 +55,7 @@ public: class Executor { public: - static void run(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) + static void run(FunctionArguments & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) { MutableColumnPtr to{block.getByPosition(result_pos).type->createColumn()}; to->reserve(input_rows_count); @@ -166,11 +166,11 @@ private: String key; }; - static std::vector prepareMoves(const char * function_name, Block & block, const ColumnNumbers & arguments, size_t first_index_argument, size_t num_index_arguments); + static std::vector prepareMoves(const char * function_name, FunctionArguments & block, const ColumnNumbers & arguments, size_t first_index_argument, size_t num_index_arguments); /// Performs moves of types MoveType::Index and MoveType::ConstIndex. template - static bool performMoves(const Block & block, const ColumnNumbers & arguments, size_t row, + static bool performMoves(const FunctionArguments & block, const ColumnNumbers & arguments, size_t row, const typename JSONParser::Element & document, const std::vector & moves, typename JSONParser::Element & element, std::string_view & last_key) { @@ -334,7 +334,7 @@ public: static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &) { @@ -362,7 +362,7 @@ public: return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers &) { return 0; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers &) { return 0; } static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view &) { @@ -386,7 +386,7 @@ public: return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -416,7 +416,7 @@ public: return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element &, const std::string_view & last_key) { @@ -450,7 +450,7 @@ public: return std::make_shared>(values); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -492,7 +492,7 @@ public: return std::make_shared>(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -557,7 +557,7 @@ public: return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -582,7 +582,7 @@ public: return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -909,9 +909,9 @@ public: return DataTypeFactory::instance().get(col_type_const->getValue()); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 2; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 2; } - void prepare(const char * function_name, const Block & block, const ColumnNumbers &, size_t result_pos) + void prepare(const char * function_name, const FunctionArguments & block, const ColumnNumbers &, size_t result_pos) { extract_tree = JSONExtractTree::build(function_name, block.getByPosition(result_pos).type); } @@ -950,9 +950,9 @@ public: return std::make_unique(tuple_type); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 2; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 2; } - void prepare(const char * function_name, const Block & block, const ColumnNumbers &, size_t result_pos) + void prepare(const char * function_name, const FunctionArguments & block, const ColumnNumbers &, size_t result_pos) { const auto & result_type = block.getByPosition(result_pos).type; const auto tuple_type = typeid_cast(result_type.get())->getNestedType(); @@ -1002,7 +1002,7 @@ public: return std::make_shared(); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -1106,7 +1106,7 @@ public: return std::make_shared(std::make_shared()); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } static bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { @@ -1138,7 +1138,7 @@ public: return std::make_unique(tuple_type); } - static size_t getNumberOfIndexArguments(const Block &, const ColumnNumbers & arguments) { return arguments.size() - 1; } + static size_t getNumberOfIndexArguments(const FunctionArguments &, const ColumnNumbers & arguments) { return arguments.size() - 1; } bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) { diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 3fc96cc1d92..ecef322052b 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -554,7 +554,7 @@ DataTypePtr FunctionUnaryLogical::getReturnTypeImpl(const DataTypes } template