From d4c1faad3b4399f19573a216a3444add70bac24f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 13 Jun 2024 11:24:00 -0300 Subject: [PATCH 001/148] testing the waters --- src/Access/Authentication.cpp | 264 ++++++++---------- src/Access/Authentication.h | 2 +- src/Access/IAccessStorage.cpp | 15 +- src/Access/LDAPAccessStorage.cpp | 4 +- src/Access/User.cpp | 3 +- src/Access/User.h | 2 +- src/Access/UsersConfigAccessStorage.cpp | 41 +-- .../Access/InterpreterCreateUserQuery.cpp | 22 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 5 +- src/Interpreters/Session.cpp | 2 +- src/Interpreters/SessionLog.cpp | 9 +- src/Parsers/Access/ASTCreateUserQuery.h | 1 + src/Parsers/Access/ParserCreateUserQuery.cpp | 17 ++ src/Parsers/CommonParsers.h | 2 + src/Storages/System/StorageSystemUsers.cpp | 2 +- 15 files changed, 208 insertions(+), 183 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index bf1fe3feec3..d7d656b703e 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -85,10 +85,26 @@ namespace #endif } +static std::vector getAuthenticationMethodsOfType( + const std::vector & authentication_methods, + const std::unordered_set & types) +{ + std::vector authentication_methods_of_type; + + for (const auto & authentication_method : authentication_methods) + { + if (types.contains(authentication_method.getType())) + { + authentication_methods_of_type.push_back(authentication_method); + } + } + + return authentication_methods_of_type; +} bool Authentication::areCredentialsValid( const Credentials & credentials, - const AuthenticationData & auth_data, + const std::vector & authentication_methods, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) { @@ -97,181 +113,149 @@ bool Authentication::areCredentialsValid( if (const auto * gss_acceptor_context = typeid_cast(&credentials)) { - switch (auth_data.getType()) + auto kerberos_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::KERBEROS}); + + for (const auto & kerberos_authentication : kerberos_authentication_methods) { - case AuthenticationType::NO_PASSWORD: - case AuthenticationType::PLAINTEXT_PASSWORD: - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::KERBEROS: - return external_authenticators.checkKerberosCredentials(auth_data.getKerberosRealm(), *gss_acceptor_context); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::MAX: - break; + if (external_authenticators.checkKerberosCredentials(kerberos_authentication.getKerberosRealm(), *gss_acceptor_context)) + { + return true; + } } + + return false; } if (const auto * mysql_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) + auto mysql_authentication_methods = getAuthenticationMethodsOfType( + authentication_methods, + {AuthenticationType::PLAINTEXT_PASSWORD, AuthenticationType::DOUBLE_SHA1_PASSWORD}); + + for (const auto & mysql_authentication_method : mysql_authentication_methods) { - case AuthenticationType::NO_PASSWORD: - return true; // N.B. even if the password is not empty! + switch (mysql_authentication_method.getType()) + { + case AuthenticationType::PLAINTEXT_PASSWORD: + if (checkPasswordPlainTextMySQL( + mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), mysql_authentication_method.getPasswordHashBinary())) + { + return true; + } + break; - case AuthenticationType::PLAINTEXT_PASSWORD: - return checkPasswordPlainTextMySQL(mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - return checkPasswordDoubleSHA1MySQL(mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::KERBEROS: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::MAX: - break; + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + if (checkPasswordDoubleSHA1MySQL( + mysql_credentials->getScramble(), + mysql_credentials->getScrambledPassword(), + mysql_authentication_method.getPasswordHashBinary())) + { + return true; + } + break; + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "something bad happened"); + } } + + return false; } if (const auto * basic_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) + auto basic_credentials_authentication_methods = getAuthenticationMethodsOfType( + authentication_methods, + {AuthenticationType::PLAINTEXT_PASSWORD, AuthenticationType::SHA256_PASSWORD, + AuthenticationType::DOUBLE_SHA1_PASSWORD, AuthenticationType::LDAP, AuthenticationType::BCRYPT_PASSWORD, + AuthenticationType::HTTP}); + + for (const auto & auth_method : basic_credentials_authentication_methods) { - case AuthenticationType::NO_PASSWORD: - return true; // N.B. even if the password is not empty! + switch (auth_method.getType()) + { + case AuthenticationType::PLAINTEXT_PASSWORD: + if (checkPasswordPlainText(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) + { + return true; + } + break; - case AuthenticationType::PLAINTEXT_PASSWORD: - return checkPasswordPlainText(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::SHA256_PASSWORD: - return checkPasswordSHA256(basic_credentials->getPassword(), auth_data.getPasswordHashBinary(), auth_data.getSalt()); - - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - return checkPasswordDoubleSHA1(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::LDAP: - return external_authenticators.checkLDAPCredentials(auth_data.getLDAPServerName(), *basic_credentials); - - case AuthenticationType::KERBEROS: - throw Authentication::Require(auth_data.getKerberosRealm()); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::BCRYPT_PASSWORD: - return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary()); - - case AuthenticationType::HTTP: - switch (auth_data.getHTTPAuthenticationScheme()) - { - case HTTPAuthenticationScheme::BASIC: + case AuthenticationType::SHA256_PASSWORD: + if (checkPasswordSHA256(basic_credentials->getPassword(), auth_method.getPasswordHashBinary(), auth_method.getSalt())) + { + return true; + } + break; + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + if (checkPasswordDoubleSHA1(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) + { + return true; + } + break; + case AuthenticationType::LDAP: + if (external_authenticators.checkLDAPCredentials(auth_method.getLDAPServerName(), *basic_credentials)) + { + return true; + } + break; + case AuthenticationType::BCRYPT_PASSWORD: + if (checkPasswordBcrypt(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) + { + return true; + } + break; + case AuthenticationType::HTTP: + if (auth_method.getHTTPAuthenticationScheme() == HTTPAuthenticationScheme::BASIC) + { return external_authenticators.checkHTTPBasicCredentials( - auth_data.getHTTPAuthenticationServerName(), *basic_credentials, settings); - } - - case AuthenticationType::MAX: - break; + auth_method.getHTTPAuthenticationServerName(), *basic_credentials, settings); + } + break; + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "something bad happened"); + } } + + return false; } if (const auto * ssl_certificate_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) + const auto ssl_certificate_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSL_CERTIFICATE}); + + for (const auto & auth_method : ssl_certificate_authentication_methods) { - case AuthenticationType::NO_PASSWORD: - case AuthenticationType::PLAINTEXT_PASSWORD: - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::KERBEROS: - throw Authentication::Require(auth_data.getKerberosRealm()); - - case AuthenticationType::SSL_CERTIFICATE: - return auth_data.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName()); - - case AuthenticationType::SSH_KEY: -#if USE_SSH - throw Authentication::Require("SSH Keys Authentication"); -#else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); -#endif - - case AuthenticationType::MAX: - break; + if (auth_method.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName())) + { + return true; + } } + + return false; } #if USE_SSH if (const auto * ssh_credentials = typeid_cast(&credentials)) { - switch (auth_data.getType()) + const auto ssh_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSL_CERTIFICATE}); + + for (const auto & auth_method : ssh_authentication_methods) { - case AuthenticationType::NO_PASSWORD: - case AuthenticationType::PLAINTEXT_PASSWORD: - case AuthenticationType::SHA256_PASSWORD: - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - case AuthenticationType::BCRYPT_PASSWORD: - case AuthenticationType::LDAP: - case AuthenticationType::HTTP: - throw Authentication::Require("ClickHouse Basic Authentication"); - - case AuthenticationType::KERBEROS: - throw Authentication::Require(auth_data.getKerberosRealm()); - - case AuthenticationType::SSL_CERTIFICATE: - throw Authentication::Require("ClickHouse X.509 Authentication"); - - case AuthenticationType::SSH_KEY: - return checkSshSignature(auth_data.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal()); - case AuthenticationType::MAX: - break; + if (checkSshSignature(auth_method.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal())) + { + return true; + } } + + return false; } #endif if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "areCredentialsValid(): authentication type {} not supported", toString(auth_data.getType())); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "TODO arthur, list possible types"); +// throw Exception(ErrorCodes::NOT_IMPLEMENTED, "areCredentialsValid(): authentication type {} not supported", toString(auth_data.getType())); } } diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index ffc497cc442..13bd487f21a 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -24,7 +24,7 @@ struct Authentication /// returned by the authentication server static bool areCredentialsValid( const Credentials & credentials, - const AuthenticationData & auth_data, + const std::vector & authentication_methods, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings); diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8d4e7d3073e..1b8c1a7f2c2 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -514,8 +514,8 @@ std::optional IAccessStorage::authenticateImpl( const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, - bool allow_no_password, - bool allow_plaintext_password) const + bool , + bool ) const { if (auto id = find(credentials.getUserName())) { @@ -525,10 +525,11 @@ std::optional IAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); - auto auth_type = user->auth_data.getType(); - if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || - ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) - throwAuthenticationTypeNotAllowed(auth_type); + // todo arthur +// auto auth_type = user->auth_data.getType(); +// if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || +// ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) +// throwAuthenticationTypeNotAllowed(auth_type); if (!areCredentialsValid(*user, credentials, external_authenticators, auth_result.settings)) throwInvalidCredentials(); @@ -564,7 +565,7 @@ bool IAccessStorage::areCredentialsValid( return false; } - return Authentication::areCredentialsValid(credentials, user.auth_data, external_authenticators, settings); + return Authentication::areCredentialsValid(credentials, user.authentication_methods, external_authenticators, settings); } diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 3206b20b691..89920878fef 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -468,8 +468,8 @@ std::optional LDAPAccessStorage::authenticateImpl( // User does not exist, so we create one, and will add it if authentication is successful. new_user = std::make_shared(); new_user->setName(credentials.getUserName()); - new_user->auth_data = AuthenticationData(AuthenticationType::LDAP); - new_user->auth_data.setLDAPServerName(ldap_server_name); + new_user->authentication_methods.emplace_back(AuthenticationType::LDAP); + new_user->authentication_methods.back().setLDAPServerName(ldap_server_name); user = new_user; } diff --git a/src/Access/User.cpp b/src/Access/User.cpp index 6a296706baf..90c151a32b7 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -16,7 +16,8 @@ bool User::equal(const IAccessEntity & other) const if (!IAccessEntity::equal(other)) return false; const auto & other_user = typeid_cast(other); - return (auth_data == other_user.auth_data) && (allowed_client_hosts == other_user.allowed_client_hosts) + return (authentication_methods == other_user.authentication_methods) + && (allowed_client_hosts == other_user.allowed_client_hosts) && (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles) && (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database) && (valid_until == other_user.valid_until); diff --git a/src/Access/User.h b/src/Access/User.h index e4ab654dafd..28f16a76b0c 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -15,7 +15,7 @@ namespace DB */ struct User : public IAccessEntity { - AuthenticationData auth_data; + std::vector authentication_methods; AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{}; AccessRights access; GrantedRoles granted_roles; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 1f9a977bab6..a4ab5549e8c 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -154,18 +154,18 @@ namespace if (has_password_plaintext) { - user->auth_data = AuthenticationData{AuthenticationType::PLAINTEXT_PASSWORD}; - user->auth_data.setPassword(config.getString(user_config + ".password")); + user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD); + user->authentication_methods.back().setPassword(config.getString(user_config + ".password")); } else if (has_password_sha256_hex) { - user->auth_data = AuthenticationData{AuthenticationType::SHA256_PASSWORD}; - user->auth_data.setPasswordHashHex(config.getString(user_config + ".password_sha256_hex")); + user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD); + user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex")); } else if (has_password_double_sha1_hex) { - user->auth_data = AuthenticationData{AuthenticationType::DOUBLE_SHA1_PASSWORD}; - user->auth_data.setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex")); + user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD); + user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex")); } else if (has_ldap) { @@ -177,19 +177,19 @@ namespace if (ldap_server_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "LDAP server name cannot be empty for user {}.", user_name); - user->auth_data = AuthenticationData{AuthenticationType::LDAP}; - user->auth_data.setLDAPServerName(ldap_server_name); + user->authentication_methods.emplace_back(AuthenticationType::LDAP); + user->authentication_methods.back().setLDAPServerName(ldap_server_name); } else if (has_kerberos) { const auto realm = config.getString(user_config + ".kerberos.realm", ""); - user->auth_data = AuthenticationData{AuthenticationType::KERBEROS}; - user->auth_data.setKerberosRealm(realm); + user->authentication_methods.emplace_back(AuthenticationType::KERBEROS); + user->authentication_methods.back().setKerberosRealm(realm); } else if (has_certificates) { - user->auth_data = AuthenticationData{AuthenticationType::SSL_CERTIFICATE}; + user->authentication_methods.emplace_back(AuthenticationType::SSL_CERTIFICATE); /// Fill list of allowed certificates. Poco::Util::AbstractConfiguration::Keys keys; @@ -205,12 +205,12 @@ namespace else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown certificate pattern type: {}", key); } - user->auth_data.setSSLCertificateCommonNames(std::move(common_names)); + user->authentication_methods.back().setSSLCertificateCommonNames(std::move(common_names)); } else if (has_ssh_keys) { #if USE_SSH - user->auth_data = AuthenticationData{AuthenticationType::SSH_KEY}; + user->authentication_methods.emplace_back(AuthenticationType::SSH_KEY); Poco::Util::AbstractConfiguration::Keys entries; config.keys(ssh_keys_config, entries); @@ -247,20 +247,25 @@ namespace else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown ssh_key entry pattern type: {}", entry); } - user->auth_data.setSSHKeys(std::move(keys)); + user->authentication_methods.back().setSSHKeys(std::move(keys)); #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh"); #endif } else if (has_http_auth) { - user->auth_data = AuthenticationData{AuthenticationType::HTTP}; - user->auth_data.setHTTPAuthenticationServerName(config.getString(http_auth_config + ".server")); + user->authentication_methods.emplace_back(AuthenticationType::HTTP); + user->authentication_methods.back().setHTTPAuthenticationServerName(config.getString(http_auth_config + ".server")); auto scheme = config.getString(http_auth_config + ".scheme"); - user->auth_data.setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); + user->authentication_methods.back().setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme)); + } + else + { + // remember to clean it up.. do I need to? smth in my memory tells me I don't need to + user->authentication_methods.emplace_back(); } - auto auth_type = user->auth_data.getType(); + auto auth_type = user->authentication_methods.back().getType(); if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) { diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 32c51b745c7..274017903ee 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -39,6 +39,7 @@ namespace const std::optional & override_settings, const std::optional & override_grantees, const std::optional & valid_until, + bool reset_authentication_methods, bool allow_implicit_no_password, bool allow_no_password, bool allow_plaintext_password) @@ -57,11 +58,22 @@ namespace "in the server configuration"); if (auth_data) - user.auth_data = *auth_data; + { + user.authentication_methods.push_back(*auth_data); + } + + if (reset_authentication_methods) + { + // todo check if element exists + auto primary_authentication_method = user.authentication_methods.back(); + user.authentication_methods.clear(); + user.authentication_methods.push_back(primary_authentication_method); + } if (auth_data || !query.alter) { - auto auth_type = user.auth_data.getType(); + // todo arthur + auto auth_type = user.authentication_methods[0].getType(); if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) { @@ -196,7 +208,7 @@ BlockIO InterpreterCreateUserQuery::execute() auto updated_user = typeid_cast>(entity->clone()); updateUserFromQueryImpl( *updated_user, query, auth_data, {}, default_roles_from_query, settings_from_query, grantees_from_query, - valid_until, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + valid_until, query.reset_authentication_methods_to_new, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); return updated_user; }; @@ -216,7 +228,7 @@ BlockIO InterpreterCreateUserQuery::execute() auto new_user = std::make_shared(); updateUserFromQueryImpl( *new_user, query, auth_data, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, - valid_until, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + valid_until, query.reset_authentication_methods_to_new, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); new_users.emplace_back(std::move(new_user)); } @@ -259,7 +271,7 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat if (query.auth_data) auth_data = AuthenticationData::fromAST(*query.auth_data, {}, !query.attach); - updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, {}, allow_no_password, allow_plaintext_password, true); + updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, {}, query.reset_authentication_methods_to_new, allow_no_password, allow_plaintext_password, true); } void registerInterpreterCreateUserQuery(InterpreterFactory & factory) diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 96d8e55a74c..e0c5abbb877 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -64,8 +64,9 @@ namespace query->default_roles = user.default_roles.toASTWithNames(*access_control); } - if (user.auth_data.getType() != AuthenticationType::NO_PASSWORD) - query->auth_data = user.auth_data.toAST(); + // todo arthur + if (user.authentication_methods[0].getType() != AuthenticationType::NO_PASSWORD) + query->auth_data = user.authentication_methods[0].toAST(); if (user.valid_until) { diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 396562189e0..0c230c10388 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -309,7 +309,7 @@ Session::~Session() AuthenticationType Session::getAuthenticationType(const String & user_name) const { - return global_context->getAccessControl().read(user_name)->auth_data.getType(); + return global_context->getAccessControl().read(user_name)->authentication_methods.back().getType(); } AuthenticationType Session::getAuthenticationTypeOrLogInFailure(const String & user_name) const diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index adb94cae0c2..51fc83f515f 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -220,9 +220,10 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, if (login_user) { log_entry.user = login_user->getName(); - log_entry.user_identified_with = login_user->auth_data.getType(); + log_entry.user_identified_with = login_user->authentication_methods.back().getType(); +// log_entry.user_identified_with = login_user->auth_data.getType(); } - log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : ""; + log_entry.external_auth_server = login_user ? login_user->authentication_methods.back().getLDAPServerName() : ""; log_entry.session_id = session_id; @@ -260,9 +261,9 @@ void SessionLog::addLogOut(const UUID & auth_id, const UserPtr & login_user, con if (login_user) { log_entry.user = login_user->getName(); - log_entry.user_identified_with = login_user->auth_data.getType(); + log_entry.user_identified_with = login_user->authentication_methods.back().getType(); } - log_entry.external_auth_server = login_user ? login_user->auth_data.getLDAPServerName() : ""; + log_entry.external_auth_server = login_user ? login_user->authentication_methods.back().getLDAPServerName() : ""; log_entry.client_info = client_info; add(std::move(log_entry)); diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 4e14d86c425..3f79f09984e 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -42,6 +42,7 @@ public: bool if_exists = false; bool if_not_exists = false; bool or_replace = false; + bool reset_authentication_methods_to_new = false; std::shared_ptr names; std::optional new_name; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index d4729ab796a..e4717572b79 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -48,6 +48,7 @@ namespace { return IParserBase::wrapParseImpl(pos, [&] { + ParserKeyword{Keyword::ADD_NEW_AUTHENTICATION_METHOD}.ignore(pos, expected); if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) { auth_data = std::make_shared(); @@ -401,6 +402,14 @@ namespace return until_p.parse(pos, valid_until, expected); }); } + + bool parseResetAuthenticationMethods(IParserBase::Pos & pos, Expected & expected) + { + return IParserBase::wrapParseImpl(pos, [&] + { + return ParserKeyword{Keyword::RESET_AUTHENTICATION_METHODS_TO_NEW}.ignore(pos, expected); + }); + } } @@ -454,6 +463,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ASTPtr valid_until; String cluster; String storage_name; + std::optional reset_authentication_methods_to_new; while (true) { @@ -467,6 +477,12 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec } } + if (!reset_authentication_methods_to_new.has_value()) + { + reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); + continue; + } + if (!valid_until) { parseValidUntil(pos, expected, valid_until); @@ -564,6 +580,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->default_database = std::move(default_database); query->valid_until = std::move(valid_until); query->storage_name = std::move(storage_name); + query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); if (query->auth_data) query->children.push_back(query->auth_data); diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index f0cbe42da80..a7f33080588 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -15,6 +15,7 @@ namespace DB MR_MACROS(ADD_PROJECTION, "ADD PROJECTION") \ MR_MACROS(ADD_STATISTICS, "ADD STATISTICS") \ MR_MACROS(ADD, "ADD") \ + MR_MACROS(ADD_NEW_AUTHENTICATION_METHOD, "ADD NEW AUTHENTICATION METHOD") \ MR_MACROS(ADMIN_OPTION_FOR, "ADMIN OPTION FOR") \ MR_MACROS(AFTER, "AFTER") \ MR_MACROS(ALGORITHM, "ALGORITHM") \ @@ -401,6 +402,7 @@ namespace DB MR_MACROS(REPLACE_PARTITION, "REPLACE PARTITION") \ MR_MACROS(REPLACE, "REPLACE") \ MR_MACROS(RESET_SETTING, "RESET SETTING") \ + MR_MACROS(RESET_AUTHENTICATION_METHODS_TO_NEW, "RESET AUTHENTICATION METHODS TO NEW") \ MR_MACROS(RESPECT_NULLS, "RESPECT NULLS") \ MR_MACROS(RESTORE, "RESTORE") \ MR_MACROS(RESTRICT, "RESTRICT") \ diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index 0c34f04844d..e8557efa9f9 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -231,7 +231,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->auth_data, user->allowed_client_hosts, + add_row(user->getName(), id, storage->getStorageName(), user->authentication_methods.back(), user->allowed_client_hosts, user->default_roles, user->grantees, user->default_database); } } From b22776d3a81784daf23ac6e04871912249695030 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 14 Jun 2024 09:34:10 -0300 Subject: [PATCH 002/148] throw exception upon auth --- src/Access/Authentication.cpp | 22 +++++++++++++++++-- src/Access/IAccessStorage.cpp | 18 ++++++++++----- .../Access/InterpreterCreateUserQuery.cpp | 4 ++-- ...InterpreterShowCreateAccessEntityQuery.cpp | 2 ++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index d7d656b703e..166230c52eb 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -254,8 +254,26 @@ bool Authentication::areCredentialsValid( if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "TODO arthur, list possible types"); -// throw Exception(ErrorCodes::NOT_IMPLEMENTED, "areCredentialsValid(): authentication type {} not supported", toString(auth_data.getType())); + + // below code sucks, but works for now I guess. + std::string possible_authentication_types; + bool first = true; + + for (const auto & authentication_method : authentication_methods) + { + if (first) + { + possible_authentication_types += ", "; + first = false; + } + possible_authentication_types += toString(authentication_method.getType()); + } + + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "areCredentialsValid(): Invalid credentials provided, available authentication methods are {}", + possible_authentication_types); + } } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 1b8c1a7f2c2..40041e62aa4 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -514,8 +514,8 @@ std::optional IAccessStorage::authenticateImpl( const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, - bool , - bool ) const + bool allow_no_password, + bool allow_plaintext_password) const { if (auto id = find(credentials.getUserName())) { @@ -526,10 +526,16 @@ std::optional IAccessStorage::authenticateImpl( throwAddressNotAllowed(address); // todo arthur -// auto auth_type = user->auth_data.getType(); -// if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || -// ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) -// throwAuthenticationTypeNotAllowed(auth_type); + // for now, just throw exception in case a user exists with invalid auth method + // back in the day, it would also throw an exception. There might be a smarter alternative + // like a user scan during startup. + for (const auto & auth_method : user->authentication_methods) + { + auto auth_type = auth_method.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + throwAuthenticationTypeNotAllowed(auth_type); + } if (!areCredentialsValid(*user, credentials, external_authenticators, auth_result.settings)) throwInvalidCredentials(); diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 274017903ee..d93f9d30f20 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -72,8 +72,8 @@ namespace if (auth_data || !query.alter) { - // todo arthur - auto auth_type = user.authentication_methods[0].getType(); + // I suppose it is guaranteed a user will always have at least one authentication method + auto auth_type = user.authentication_methods.back().getType(); if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) { diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index e0c5abbb877..7de939afa5a 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -65,6 +65,8 @@ namespace } // todo arthur + // to fix this, I'll need to turn `query->auth_data` into a list + // that also means creating a user with multiple authentication methods should be allowed if (user.authentication_methods[0].getType() != AuthenticationType::NO_PASSWORD) query->auth_data = user.authentication_methods[0].toAST(); From 98e5ea52062048a00c8a1cc9e53c376717b88e72 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 14 Jun 2024 10:02:40 -0300 Subject: [PATCH 003/148] style fix --- src/Access/Authentication.cpp | 3 ++- src/Interpreters/SessionLog.cpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 166230c52eb..80153f415a0 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -16,7 +16,7 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; - extern const int SUPPORT_IS_DISABLED; + extern const int LOGICAL_ERROR; } namespace @@ -256,6 +256,7 @@ bool Authentication::areCredentialsValid( // below code sucks, but works for now I guess. + // might be a problem if no auth method has been registered std::string possible_authentication_types; bool first = true; diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index 51fc83f515f..903eaff499f 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -207,6 +207,7 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length()); } +// todo arthur fix this method void SessionLog::addLoginSuccess(const UUID & auth_id, const String & session_id, const Settings & settings, From c1250ccb35cd438d06445f82c096dde44f627188 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 14 Jun 2024 10:58:57 -0300 Subject: [PATCH 004/148] append default constructed auth method upon alter without auth data --- src/Access/Authentication.cpp | 152 +++++++++++------- .../Access/InterpreterCreateUserQuery.cpp | 5 + 2 files changed, 98 insertions(+), 59 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 80153f415a0..05140fcb8d5 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -83,38 +83,30 @@ namespace return false; } #endif -} -static std::vector getAuthenticationMethodsOfType( - const std::vector & authentication_methods, - const std::unordered_set & types) -{ - std::vector authentication_methods_of_type; - - for (const auto & authentication_method : authentication_methods) + std::vector getAuthenticationMethodsOfType( + const std::vector & authentication_methods, + const std::unordered_set & types) { - if (types.contains(authentication_method.getType())) + std::vector authentication_methods_of_type; + + for (const auto & authentication_method : authentication_methods) { - authentication_methods_of_type.push_back(authentication_method); + if (types.contains(authentication_method.getType())) + { + authentication_methods_of_type.push_back(authentication_method); + } } + + return authentication_methods_of_type; } - return authentication_methods_of_type; -} - -bool Authentication::areCredentialsValid( - const Credentials & credentials, - const std::vector & authentication_methods, - const ExternalAuthenticators & external_authenticators, - SettingsChanges & settings) -{ - if (!credentials.isReady()) - return false; - - if (const auto * gss_acceptor_context = typeid_cast(&credentials)) + bool checkKerberosAuthentication( + const GSSAcceptorContext * gss_acceptor_context, + const std::vector & authentication_methods, + const ExternalAuthenticators & external_authenticators) { auto kerberos_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::KERBEROS}); - for (const auto & kerberos_authentication : kerberos_authentication_methods) { if (external_authenticators.checkKerberosCredentials(kerberos_authentication.getKerberosRealm(), *gss_acceptor_context)) @@ -122,11 +114,12 @@ bool Authentication::areCredentialsValid( return true; } } - return false; } - if (const auto * mysql_credentials = typeid_cast(&credentials)) + bool checkMySQLAuthentication( + const MySQLNative41Credentials * mysql_credentials, + const std::vector & authentication_methods) { auto mysql_authentication_methods = getAuthenticationMethodsOfType( authentication_methods, @@ -138,7 +131,7 @@ bool Authentication::areCredentialsValid( { case AuthenticationType::PLAINTEXT_PASSWORD: if (checkPasswordPlainTextMySQL( - mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), mysql_authentication_method.getPasswordHashBinary())) + mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), mysql_authentication_method.getPasswordHashBinary())) { return true; } @@ -154,14 +147,17 @@ bool Authentication::areCredentialsValid( } break; default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "something bad happened"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid MySQL authentication type"); } } - return false; } - if (const auto * basic_credentials = typeid_cast(&credentials)) + bool checkBasicAuthentication( + const BasicCredentials * basic_credentials, + const std::vector & authentication_methods, + const ExternalAuthenticators & external_authenticators, + SettingsChanges & settings) { auto basic_credentials_authentication_methods = getAuthenticationMethodsOfType( authentication_methods, @@ -212,17 +208,17 @@ bool Authentication::areCredentialsValid( } break; default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "something bad happened"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid basic authentication type"); } } - return false; } - if (const auto * ssl_certificate_credentials = typeid_cast(&credentials)) + bool checkSSLCertificateAuthentication( + const SSLCertificateCredentials * ssl_certificate_credentials, + const std::vector & authentication_methods) { const auto ssl_certificate_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSL_CERTIFICATE}); - for (const auto & auth_method : ssl_certificate_authentication_methods) { if (auth_method.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName())) @@ -230,15 +226,15 @@ bool Authentication::areCredentialsValid( return true; } } - return false; } #if USE_SSH - if (const auto * ssh_credentials = typeid_cast(&credentials)) + bool checkSshAuthentication( + const SshCredentials * ssh_credentials, + const std::vector & authentication_methods) { - const auto ssh_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSL_CERTIFICATE}); - + const auto ssh_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSH_KEY}); for (const auto & auth_method : ssh_authentication_methods) { if (checkSshSignature(auth_method.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal())) @@ -246,35 +242,73 @@ bool Authentication::areCredentialsValid( return true; } } - return false; } #endif + [[noreturn]] void throwInvalidCredentialsException(const std::vector & authentication_methods) + { + std::string possible_authentication_types; + bool first = true; + + for (const auto & authentication_method : authentication_methods) + { + if (!first) + { + possible_authentication_types += ", "; + } + possible_authentication_types += toString(authentication_method.getType()); + first = false; + } + + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "areCredentialsValid(): Invalid credentials provided, available authentication methods are {}", + possible_authentication_types); + } +} + +bool Authentication::areCredentialsValid( + const Credentials & credentials, + const std::vector & authentication_methods, + const ExternalAuthenticators & external_authenticators, + SettingsChanges & settings) +{ + if (!credentials.isReady()) + return false; + + if (const auto * gss_acceptor_context = typeid_cast(&credentials)) + { + return checkKerberosAuthentication(gss_acceptor_context, authentication_methods, external_authenticators); + } + + if (const auto * mysql_credentials = typeid_cast(&credentials)) + { + return checkMySQLAuthentication(mysql_credentials, authentication_methods); + } + + if (const auto * basic_credentials = typeid_cast(&credentials)) + { + return checkBasicAuthentication(basic_credentials, authentication_methods, external_authenticators, settings); + } + + if (const auto * ssl_certificate_credentials = typeid_cast(&credentials)) + { + return checkSSLCertificateAuthentication(ssl_certificate_credentials, authentication_methods); + } + +#if USE_SSH + if (const auto * ssh_credentials = typeid_cast(&credentials)) + { + return checkSshAuthentication(ssh_credentials, authentication_methods); + } +#endif + if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; - // below code sucks, but works for now I guess. - // might be a problem if no auth method has been registered - std::string possible_authentication_types; - bool first = true; - - for (const auto & authentication_method : authentication_methods) - { - if (first) - { - possible_authentication_types += ", "; - first = false; - } - possible_authentication_types += toString(authentication_method.getType()); - } - - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "areCredentialsValid(): Invalid credentials provided, available authentication methods are {}", - possible_authentication_types); - + throwInvalidCredentialsException(authentication_methods); } } diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index d93f9d30f20..398d77863d4 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -61,6 +61,11 @@ namespace { user.authentication_methods.push_back(*auth_data); } + else if (user.authentication_methods.empty()) + { + // previously, a user always had a default constructed auth method.. maybe I should put this somewhere else + user.authentication_methods.emplace_back(); + } if (reset_authentication_methods) { From 70e493322123f13b7981e20cdb9f904b0bfe28e6 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 14 Jun 2024 14:39:55 -0300 Subject: [PATCH 005/148] fix no pwd authentication --- src/Access/Authentication.cpp | 6 +++- src/Client/ClientBase.cpp | 6 ++-- .../Access/InterpreterCreateUserQuery.cpp | 36 +++++++++++++------ ...InterpreterShowCreateAccessEntityQuery.cpp | 12 ++++--- src/Parsers/Access/ASTCreateUserQuery.cpp | 20 +++++++---- src/Parsers/Access/ASTCreateUserQuery.h | 2 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 16 ++++++--- 7 files changed, 65 insertions(+), 33 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 05140fcb8d5..d486f717b92 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -161,7 +161,7 @@ namespace { auto basic_credentials_authentication_methods = getAuthenticationMethodsOfType( authentication_methods, - {AuthenticationType::PLAINTEXT_PASSWORD, AuthenticationType::SHA256_PASSWORD, + {AuthenticationType::NO_PASSWORD, AuthenticationType::PLAINTEXT_PASSWORD, AuthenticationType::SHA256_PASSWORD, AuthenticationType::DOUBLE_SHA1_PASSWORD, AuthenticationType::LDAP, AuthenticationType::BCRYPT_PASSWORD, AuthenticationType::HTTP}); @@ -169,6 +169,10 @@ namespace { switch (auth_method.getType()) { + case AuthenticationType::NO_PASSWORD: + { + return true; + } case AuthenticationType::PLAINTEXT_PASSWORD: if (checkPasswordPlainText(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) { diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index f8391c64d5a..a4692da0e53 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1903,11 +1903,11 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (const auto * create_user_query = parsed_query->as()) { - if (!create_user_query->attach && create_user_query->auth_data) + if (!create_user_query->attach && !create_user_query->auth_data.empty()) { - if (const auto * auth_data = create_user_query->auth_data->as()) + for (const auto & authentication_method : create_user_query->auth_data) { - auto password = auth_data->getPassword(); + auto password = authentication_method->getPassword(); if (password) global_context->getAccessControl().checkPasswordComplexityRules(*password); diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 398d77863d4..b49d169dda4 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -33,7 +33,7 @@ namespace void updateUserFromQueryImpl( User & user, const ASTCreateUserQuery & query, - const std::optional auth_data, + const std::vector auth_data, const std::shared_ptr & override_name, const std::optional & override_default_roles, const std::optional & override_settings, @@ -51,15 +51,19 @@ namespace else if (query.names->size() == 1) user.setName(query.names->front()->toString()); - if (!query.attach && !query.alter && !auth_data && !allow_implicit_no_password) + // todo arthur check if auth_data.empty makes sense + if (!query.attach && !query.alter && !auth_data.empty() && !allow_implicit_no_password) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication type NO_PASSWORD must " "be explicitly specified, check the setting allow_implicit_no_password " "in the server configuration"); - if (auth_data) + if (!auth_data.empty()) { - user.authentication_methods.push_back(*auth_data); + for (const auto & authentication_method : auth_data) + { + user.authentication_methods.push_back(authentication_method); + } } else if (user.authentication_methods.empty()) { @@ -75,7 +79,7 @@ namespace user.authentication_methods.push_back(primary_authentication_method); } - if (auth_data || !query.alter) + if (!auth_data.empty() || !query.alter) { // I suppose it is guaranteed a user will always have at least one authentication method auto auth_type = user.authentication_methods.back().getType(); @@ -145,9 +149,14 @@ BlockIO InterpreterCreateUserQuery::execute() bool no_password_allowed = access_control.isNoPasswordAllowed(); bool plaintext_password_allowed = access_control.isPlaintextPasswordAllowed(); - std::optional auth_data; - if (query.auth_data) - auth_data = AuthenticationData::fromAST(*query.auth_data, getContext(), !query.attach); + std::vector auth_data; + if (!query.auth_data.empty()) + { + for (const auto & authentication_method_ast : query.auth_data) + { + auth_data.push_back(AuthenticationData::fromAST(*authentication_method_ast, getContext(), !query.attach)); + } + } std::optional valid_until; if (query.valid_until) @@ -272,9 +281,14 @@ BlockIO InterpreterCreateUserQuery::execute() void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password) { - std::optional auth_data; - if (query.auth_data) - auth_data = AuthenticationData::fromAST(*query.auth_data, {}, !query.attach); + std::vector auth_data; + if (!query.auth_data.empty()) + { + for (const auto & authentication_method_ast : query.auth_data) + { + auth_data.emplace_back(AuthenticationData::fromAST(*authentication_method_ast, {}, !query.attach)); + } + } updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, {}, query.reset_authentication_methods_to_new, allow_no_password, allow_plaintext_password, true); } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 7de939afa5a..70dd859c70b 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -64,11 +64,13 @@ namespace query->default_roles = user.default_roles.toASTWithNames(*access_control); } - // todo arthur - // to fix this, I'll need to turn `query->auth_data` into a list - // that also means creating a user with multiple authentication methods should be allowed - if (user.authentication_methods[0].getType() != AuthenticationType::NO_PASSWORD) - query->auth_data = user.authentication_methods[0].toAST(); + for (const auto & authentication_method : user.authentication_methods) + { + if (authentication_method.getType() != AuthenticationType::NO_PASSWORD) + { + query->auth_data.push_back(authentication_method.toAST()); + } + } if (user.valid_until) { diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 02735568a04..975f81e57ca 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -18,9 +18,12 @@ namespace << quoteString(new_name); } - void formatAuthenticationData(const ASTAuthenticationData & auth_data, const IAST::FormatSettings & settings) + void formatAuthenticationData(const std::vector> & auth_data, const IAST::FormatSettings & settings) { - auth_data.format(settings); + for (const auto & authentication_method : auth_data) + { + authentication_method->format(settings); + } } void formatValidUntil(const IAST & valid_until, const IAST::FormatSettings & settings) @@ -180,10 +183,13 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); - if (auth_data) + if (!auth_data.empty()) { - res->auth_data = std::static_pointer_cast(auth_data->clone()); - res->children.push_back(res->auth_data); + for (const auto & authentication_method : auth_data) + { + res->auth_data.push_back(std::static_pointer_cast(authentication_method->clone())); + res->children.push_back(res->auth_data.back()); + } } return res; @@ -222,8 +228,8 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (new_name) formatRenameTo(*new_name, format); - if (auth_data) - formatAuthenticationData(*auth_data, format); + if (!auth_data.empty()) + formatAuthenticationData(auth_data, format); if (valid_until) formatValidUntil(*valid_until, format); diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 3f79f09984e..58f167fe5b7 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -48,7 +48,7 @@ public: std::optional new_name; String storage_name; - std::shared_ptr auth_data; + std::vector> auth_data; std::optional hosts; std::optional add_hosts; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index e4717572b79..1c4b23e1a34 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -455,7 +455,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec std::optional hosts; std::optional add_hosts; std::optional remove_hosts; - std::shared_ptr auth_data; + std::vector> auth_data; std::shared_ptr default_roles; std::shared_ptr settings; std::shared_ptr grantees; @@ -467,12 +467,12 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec while (true) { - if (!auth_data) + if (auth_data.empty()) { std::shared_ptr new_auth_data; if (parseAuthenticationData(pos, expected, new_auth_data)) { - auth_data = std::move(new_auth_data); + auth_data.push_back(new_auth_data); continue; } } @@ -582,8 +582,14 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->storage_name = std::move(storage_name); query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); - if (query->auth_data) - query->children.push_back(query->auth_data); + if (!query->auth_data.empty()) + { + // as of now, this will always have a single element, but looping just in case. + for (const auto & authentication_method : query->auth_data) + { + query->children.push_back(authentication_method); + } + } if (query->valid_until) query->children.push_back(query->valid_until); From 6ac24fcf54aef805b9a37509d4c27b5921d1628a Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 14 Jun 2024 18:08:01 -0300 Subject: [PATCH 006/148] fix test build --- src/Parsers/tests/gtest_Parser.cpp | 3 ++- src/Parsers/tests/gtest_common.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index f0abc68f966..fd22dd34128 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -87,7 +87,8 @@ TEST_P(ParserTest, parseQuery) { if (input_text.starts_with("ATTACH")) { - auto salt = (dynamic_cast(ast.get())->auth_data)->getSalt().value_or(""); + // todo arthur + auto salt = (dynamic_cast(ast.get())->auth_data.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } else diff --git a/src/Parsers/tests/gtest_common.cpp b/src/Parsers/tests/gtest_common.cpp index 8ff9400d8a2..bc780e17ffd 100644 --- a/src/Parsers/tests/gtest_common.cpp +++ b/src/Parsers/tests/gtest_common.cpp @@ -63,7 +63,8 @@ TEST_P(ParserKQLTest, parseKQLQuery) { if (input_text.starts_with("ATTACH")) { - auto salt = (dynamic_cast(ast.get())->auth_data)->getSalt().value_or(""); + // todo arthur check + auto salt = (dynamic_cast(ast.get())->auth_data.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } else From 1514dcbb346921509142f042ce1da8d4359d5a3e Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 19 Jun 2024 11:57:38 -0300 Subject: [PATCH 007/148] ptal --- src/Parsers/Access/ParserCreateUserQuery.cpp | 67 +++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 1c4b23e1a34..71d8e344646 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -26,6 +26,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, std::optional & new_name) @@ -48,7 +53,6 @@ namespace { return IParserBase::wrapParseImpl(pos, [&] { - ParserKeyword{Keyword::ADD_NEW_AUTHENTICATION_METHOD}.ignore(pos, expected); if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) { auth_data = std::make_shared(); @@ -403,6 +407,19 @@ namespace }); } + bool parseAddNewAuthenticationMethod(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{Keyword::ADD_NEW_AUTHENTICATION_METHOD}.ignore(pos, expected)) + { + return false; + } + + return parseAuthenticationData(pos, expected, auth_data); + }); + } + bool parseResetAuthenticationMethods(IParserBase::Pos & pos, Expected & expected) { return IParserBase::wrapParseImpl(pos, [&] @@ -465,18 +482,42 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec String storage_name; std::optional reset_authentication_methods_to_new; + bool parsed_identified_with = false; + bool parsed_add_new_method = false; + while (true) { - if (auth_data.empty()) + std::shared_ptr identified_with_auth_data; + bool parse_auth_data = parseAuthenticationData(pos, expected, identified_with_auth_data); + + bool found_multiple_identified_with = parsed_identified_with && parse_auth_data; + + if (found_multiple_identified_with) { - std::shared_ptr new_auth_data; - if (parseAuthenticationData(pos, expected, new_auth_data)) - { - auth_data.push_back(new_auth_data); - continue; - } + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only one identified with is permitted"); } + if (parse_auth_data) + { + if (parsed_add_new_method) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Identified with must precede add new auth"); + } + auth_data.push_back(identified_with_auth_data); + parsed_identified_with = true; + continue; + } + + std::shared_ptr add_new_auth_method; + parsed_add_new_method = parseAddNewAuthenticationMethod(pos, expected, add_new_auth_method); + + if (parsed_add_new_method) + { + auth_data.push_back(add_new_auth_method); + continue; + } + + // todo arthur maybe check that neither identified with or add new auth method has been parsed if (!reset_authentication_methods_to_new.has_value()) { reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); @@ -580,15 +621,11 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->default_database = std::move(default_database); query->valid_until = std::move(valid_until); query->storage_name = std::move(storage_name); - query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); + query->reset_authentication_methods_to_new = parsed_identified_with || reset_authentication_methods_to_new.value_or(false); - if (!query->auth_data.empty()) + for (const auto & authentication_method : query->auth_data) { - // as of now, this will always have a single element, but looping just in case. - for (const auto & authentication_method : query->auth_data) - { - query->children.push_back(authentication_method); - } + query->children.push_back(authentication_method); } if (query->valid_until) From 179d54505ad39245e13f40bde7c0a719744f0555 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 20 Jun 2024 15:07:16 -0300 Subject: [PATCH 008/148] make progress, seems functional --- src/Access/IAccessStorage.cpp | 1 - src/Access/UsersConfigAccessStorage.cpp | 15 ++-- src/Core/PostgreSQLProtocol.h | 19 +++--- .../Access/InterpreterCreateUserQuery.cpp | 68 ++++++++++++------- src/Interpreters/Session.cpp | 17 +++-- src/Interpreters/Session.h | 4 +- src/Parsers/Access/ASTAuthenticationData.cpp | 4 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 14 ++-- src/Parsers/Access/ASTCreateUserQuery.h | 1 + src/Parsers/Access/ParserCreateUserQuery.cpp | 5 +- src/Parsers/IAST.h | 6 +- src/Server/MySQLHandler.cpp | 13 ++-- src/Server/TCPHandler.cpp | 12 +++- 13 files changed, 119 insertions(+), 60 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 40041e62aa4..f1725dafdf4 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -525,7 +525,6 @@ std::optional IAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); - // todo arthur // for now, just throw exception in case a user exists with invalid auth method // back in the day, it would also throw an exception. There might be a smarter alternative // like a user scan during startup. diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index a4ab5549e8c..cfb02b0962b 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -265,13 +265,16 @@ namespace user->authentication_methods.emplace_back(); } - auto auth_type = user->authentication_methods.back().getType(); - if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || - ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + for (const auto & authentication_method : user->authentication_methods) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", - toString(auth_type), AuthenticationTypeInfo::get(auth_type).name); + auto auth_type = authentication_method.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", + toString(auth_type), AuthenticationTypeInfo::get(auth_type).name); + } } const auto profile_name_config = user_config + ".profile"; diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 807e4a7187a..5dc9082d49d 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -890,16 +890,19 @@ public: Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - AuthenticationType user_auth_type; try { - user_auth_type = session.getAuthenticationTypeOrLogInFailure(user_name); - if (type_to_method.find(user_auth_type) != type_to_method.end()) + const auto user_authentication_types = session.getAuthenticationTypesOrLogInFailure(user_name); + + for (auto user_authentication_type : user_authentication_types) { - type_to_method[user_auth_type]->authenticate(user_name, session, mt, address); - mt.send(Messaging::AuthenticationOk(), true); - LOG_DEBUG(log, "Authentication for user {} was successful.", user_name); - return; + if (type_to_method.find(user_authentication_type) != type_to_method.end()) + { + type_to_method[user_authentication_type]->authenticate(user_name, session, mt, address); + mt.send(Messaging::AuthenticationOk(), true); + LOG_DEBUG(log, "Authentication for user {} was successful.", user_name); + return; + } } } catch (const Exception&) @@ -913,7 +916,7 @@ public: mt.send(Messaging::ErrorOrNoticeResponse(Messaging::ErrorOrNoticeResponse::ERROR, "0A000", "Authentication method is not supported"), true); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Authentication method is not supported: {}", user_auth_type); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "None of the authentication methods registered for the user are supported"); } }; } diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index b49d169dda4..c79b7b5bc34 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -40,6 +40,7 @@ namespace const std::optional & override_grantees, const std::optional & valid_until, bool reset_authentication_methods, + bool replace_authentication_methods, bool allow_implicit_no_password, bool allow_no_password, bool allow_plaintext_password) @@ -51,45 +52,48 @@ namespace else if (query.names->size() == 1) user.setName(query.names->front()->toString()); - // todo arthur check if auth_data.empty makes sense if (!query.attach && !query.alter && !auth_data.empty() && !allow_implicit_no_password) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication type NO_PASSWORD must " "be explicitly specified, check the setting allow_implicit_no_password " "in the server configuration"); - if (!auth_data.empty()) - { - for (const auto & authentication_method : auth_data) - { - user.authentication_methods.push_back(authentication_method); - } - } - else if (user.authentication_methods.empty()) + if (user.authentication_methods.empty() && auth_data.empty()) { // previously, a user always had a default constructed auth method.. maybe I should put this somewhere else user.authentication_methods.emplace_back(); } - if (reset_authentication_methods) + if (replace_authentication_methods) { - // todo check if element exists - auto primary_authentication_method = user.authentication_methods.back(); user.authentication_methods.clear(); - user.authentication_methods.push_back(primary_authentication_method); } - if (!auth_data.empty() || !query.alter) + for (const auto & authentication_method : auth_data) { - // I suppose it is guaranteed a user will always have at least one authentication method - auto auth_type = user.authentication_methods.back().getType(); - if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || - ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + user.authentication_methods.emplace_back(authentication_method); + } + + if (reset_authentication_methods) + { + auto backup_authentication_method = user.authentication_methods.back(); + user.authentication_methods.clear(); + user.authentication_methods.emplace_back(backup_authentication_method); + } + + if (!query.alter) + { + for (const auto & authentication_method : user.authentication_methods) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", - toString(auth_type), - AuthenticationTypeInfo::get(auth_type).name); + auto auth_type = authentication_method.getType(); + if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || + ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", + toString(auth_type), + AuthenticationTypeInfo::get(auth_type).name); + } } } @@ -222,7 +226,8 @@ BlockIO InterpreterCreateUserQuery::execute() auto updated_user = typeid_cast>(entity->clone()); updateUserFromQueryImpl( *updated_user, query, auth_data, {}, default_roles_from_query, settings_from_query, grantees_from_query, - valid_until, query.reset_authentication_methods_to_new, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, + implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); return updated_user; }; @@ -242,7 +247,8 @@ BlockIO InterpreterCreateUserQuery::execute() auto new_user = std::make_shared(); updateUserFromQueryImpl( *new_user, query, auth_data, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, - valid_until, query.reset_authentication_methods_to_new, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, + implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); new_users.emplace_back(std::move(new_user)); } @@ -290,7 +296,19 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat } } - updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, {}, query.reset_authentication_methods_to_new, allow_no_password, allow_plaintext_password, true); + updateUserFromQueryImpl(user, + query, + auth_data, + {}, + {}, + {}, + {}, + {}, + query.reset_authentication_methods_to_new, + query.replace_authentication_methods, + allow_no_password, + allow_plaintext_password, + true); } void registerInterpreterCreateUserQuery(InterpreterFactory & factory) diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 0c230c10388..89b764a5788 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -307,16 +307,25 @@ Session::~Session() } } -AuthenticationType Session::getAuthenticationType(const String & user_name) const +std::unordered_set Session::getAuthenticationTypes(const String & user_name) const { - return global_context->getAccessControl().read(user_name)->authentication_methods.back().getType(); + std::unordered_set authentication_types; + + const auto user_to_query = global_context->getAccessControl().read(user_name); + + for (const auto & authentication_method : user_to_query->authentication_methods) + { + authentication_types.insert(authentication_method.getType()); + } + + return authentication_types; } -AuthenticationType Session::getAuthenticationTypeOrLogInFailure(const String & user_name) const +std::unordered_set Session::getAuthenticationTypesOrLogInFailure(const String & user_name) const { try { - return getAuthenticationType(user_name); + return getAuthenticationTypes(user_name); } catch (const Exception & e) { diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 14f6f806acd..6dac52d4a39 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -43,10 +43,10 @@ public: Session & operator=(const Session &) = delete; /// Provides information about the authentication type of a specified user. - AuthenticationType getAuthenticationType(const String & user_name) const; + std::unordered_set getAuthenticationTypes(const String & user_name) const; /// Same as getAuthenticationType, but adds LoginFailure event in case of error. - AuthenticationType getAuthenticationTypeOrLogInFailure(const String & user_name) const; + std::unordered_set getAuthenticationTypesOrLogInFailure(const String & user_name) const; /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. /// The function throws an exception if there is no such user or password is wrong. diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 3a62480dc0c..c155c588a24 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -154,7 +154,9 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = AuthenticationTypeInfo::get(*type).name; } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); + const char * identified_string = settings.additional_authentication_method ? " ADD NEW AUTHENTICATION METHOD IDENTIFIED" : " IDENTIFIED"; + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << identified_string << (settings.hilite ? IAST::hilite_none : ""); if (!auth_type_name.empty()) { diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 975f81e57ca..2e6014f0d59 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -20,9 +20,14 @@ namespace void formatAuthenticationData(const std::vector> & auth_data, const IAST::FormatSettings & settings) { - for (const auto & authentication_method : auth_data) + auth_data[0]->format(settings); + + auto settings_with_additional_authentication_method = settings; + settings_with_additional_authentication_method.additional_authentication_method = true; + + for (auto i = 1u; i < auth_data.size(); i++) { - authentication_method->format(settings); + auth_data[i]->format(settings_with_additional_authentication_method); } } @@ -187,8 +192,9 @@ ASTPtr ASTCreateUserQuery::clone() const { for (const auto & authentication_method : auth_data) { - res->auth_data.push_back(std::static_pointer_cast(authentication_method->clone())); - res->children.push_back(res->auth_data.back()); + auto ast_clone = std::static_pointer_cast(authentication_method->clone()); + res->auth_data.push_back(ast_clone); + res->children.push_back(ast_clone); } } diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 58f167fe5b7..24af6d00ca4 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -43,6 +43,7 @@ public: bool if_not_exists = false; bool or_replace = false; bool reset_authentication_methods_to_new = false; + bool replace_authentication_methods = false; std::shared_ptr names; std::optional new_name; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 71d8e344646..213360fface 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -517,11 +517,9 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec continue; } - // todo arthur maybe check that neither identified with or add new auth method has been parsed if (!reset_authentication_methods_to_new.has_value()) { reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); - continue; } if (!valid_until) @@ -621,7 +619,8 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->default_database = std::move(default_database); query->valid_until = std::move(valid_until); query->storage_name = std::move(storage_name); - query->reset_authentication_methods_to_new = parsed_identified_with || reset_authentication_methods_to_new.value_or(false); + query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); + query->replace_authentication_methods = parsed_identified_with; for (const auto & authentication_method : query->auth_data) { diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index ee70fed0f07..a74f321eead 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -201,6 +201,7 @@ public: bool show_secrets; /// Show secret parts of the AST (e.g. passwords, encryption keys). char nl_or_ws; /// Newline or whitespace. LiteralEscapingStyle literal_escaping_style; + bool additional_authentication_method; explicit FormatSettings( WriteBuffer & ostr_, @@ -209,7 +210,8 @@ public: bool always_quote_identifiers_ = false, IdentifierQuotingStyle identifier_quoting_style_ = IdentifierQuotingStyle::Backticks, bool show_secrets_ = true, - LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular) + LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular, + bool additional_authentication_method_ = false) : ostr(ostr_) , one_line(one_line_) , hilite(hilite_) @@ -218,6 +220,7 @@ public: , show_secrets(show_secrets_) , nl_or_ws(one_line ? ' ' : '\n') , literal_escaping_style(literal_escaping_style_) + , additional_authentication_method(additional_authentication_method_) { } @@ -230,6 +233,7 @@ public: , show_secrets(other.show_secrets) , nl_or_ws(other.nl_or_ws) , literal_escaping_style(other.literal_escaping_style) + , additional_authentication_method(other.additional_authentication_method) { } diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 6456f6d24ff..86f3dba1697 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -373,11 +373,16 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl { try { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible - // (if password is specified using double SHA1). Otherwise, SHA256 plugin is used. - if (session->getAuthenticationTypeOrLogInFailure(user_name) == DB::AuthenticationType::SHA256_PASSWORD) + const auto user_authentication_types = session->getAuthenticationTypesOrLogInFailure(user_name); + + for (const auto user_authentication_type : user_authentication_types) { - authPluginSSL(); + // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible + // (if password is specified using double SHA1). Otherwise, SHA256 plugin is used. + if (user_authentication_type == DB::AuthenticationType::SHA256_PASSWORD) + { + authPluginSSL(); + } } std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e3a820340ad..073e8d6cb5f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1498,7 +1498,17 @@ void TCPHandler::receiveHello() /// Perform handshake for SSH authentication if (is_ssh_based_auth) { - if (session->getAuthenticationTypeOrLogInFailure(user) != AuthenticationType::SSH_KEY) + const auto authentication_types = session->getAuthenticationTypesOrLogInFailure(user); + + bool user_supports_ssh_authentication = std::find_if( + authentication_types.begin(), + authentication_types.end(), + [](auto authentication_type) + { + return authentication_type == AuthenticationType::SSH_KEY; + }) != authentication_types.end(); + + if (!user_supports_ssh_authentication) throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Expected authentication with SSH key"); if (client_tcp_protocol_version < DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION) From a1211a0f5a66a676d7742dc102bb8a223945057d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 20 Jun 2024 16:51:52 -0300 Subject: [PATCH 009/148] simplify syntax --- src/Parsers/Access/ASTAuthenticationData.cpp | 2 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 2 +- src/Parsers/CommonParsers.h | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index c155c588a24..73fa5a5a692 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -154,7 +154,7 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = AuthenticationTypeInfo::get(*type).name; } - const char * identified_string = settings.additional_authentication_method ? " ADD NEW AUTHENTICATION METHOD IDENTIFIED" : " IDENTIFIED"; + const char * identified_string = settings.additional_authentication_method ? " ADD IDENTIFIED" : " IDENTIFIED"; settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << identified_string << (settings.hilite ? IAST::hilite_none : ""); diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 213360fface..36162086609 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -411,7 +411,7 @@ namespace { return IParserBase::wrapParseImpl(pos, [&] { - if (!ParserKeyword{Keyword::ADD_NEW_AUTHENTICATION_METHOD}.ignore(pos, expected)) + if (!ParserKeyword{Keyword::ADD}.ignore(pos, expected)) { return false; } diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index a7f33080588..726e491fbf3 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -15,7 +15,6 @@ namespace DB MR_MACROS(ADD_PROJECTION, "ADD PROJECTION") \ MR_MACROS(ADD_STATISTICS, "ADD STATISTICS") \ MR_MACROS(ADD, "ADD") \ - MR_MACROS(ADD_NEW_AUTHENTICATION_METHOD, "ADD NEW AUTHENTICATION METHOD") \ MR_MACROS(ADMIN_OPTION_FOR, "ADMIN OPTION FOR") \ MR_MACROS(AFTER, "AFTER") \ MR_MACROS(ALGORITHM, "ALGORITHM") \ From 55da169fe737ec6c2ca567398bf2a4b3997375d3 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 21 Jun 2024 10:30:12 -0300 Subject: [PATCH 010/148] fix wrong condition --- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index c79b7b5bc34..bc507b5f156 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -52,7 +52,7 @@ namespace else if (query.names->size() == 1) user.setName(query.names->front()->toString()); - if (!query.attach && !query.alter && !auth_data.empty() && !allow_implicit_no_password) + if (!query.attach && !query.alter && auth_data.empty() && !allow_implicit_no_password) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication type NO_PASSWORD must " "be explicitly specified, check the setting allow_implicit_no_password " From 08c9cc18d6a0e07094871b90a8f88088d96c3454 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 21 Jun 2024 12:05:58 -0300 Subject: [PATCH 011/148] fix astcreateuserquery clone --- src/Parsers/Access/ASTCreateUserQuery.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 2e6014f0d59..0d5db681a7e 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -172,6 +172,7 @@ ASTPtr ASTCreateUserQuery::clone() const { auto res = std::make_shared(*this); res->children.clear(); + res->auth_data.clear(); if (names) res->names = std::static_pointer_cast(names->clone()); @@ -188,6 +189,7 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); + // this is weird. if (!auth_data.empty()) { for (const auto & authentication_method : auth_data) From 237abda2eb2c6923835e93916b3e44a1c9e33299 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 21 Jun 2024 15:35:50 -0300 Subject: [PATCH 012/148] add some ut --- src/Parsers/tests/gtest_Parser.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index fd22dd34128..3b4cd9af695 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -284,6 +284,10 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, + { + "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'", + "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'" + }, { "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'" @@ -292,6 +296,10 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, + { + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'", + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'" + }, { "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'" @@ -299,6 +307,10 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, { "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123' SALT 'EFFD7F6B03B3EA68B8F86C1E91614DD50E42EB31EF7160524916444D58B5E264'", "throws Syntax error" + }, + { + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' IDENTIFIED WITH plaintext_password BY 'def123'", + "throws Only one identified with is permitted" } }))); From fe0d3b3e27a5a70337b4d9174e1244569b4b0043 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 15:48:46 -0300 Subject: [PATCH 013/148] initial tests --- .../03174_add_identified_with.reference | 38 ++++++++ .../0_stateless/03174_add_identified_with.sh | 92 +++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 tests/queries/0_stateless/03174_add_identified_with.reference create mode 100755 tests/queries/0_stateless/03174_add_identified_with.sh diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference new file mode 100644 index 00000000000..bd855d06ffc --- /dev/null +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -0,0 +1,38 @@ +Basic authentication after user creation +1 +Changed password, old password should not work +AUTHENTICATION_FAILED +New password should work +1 +Two new passwords were added, should both work +1 +1 +Gen ssh key +Authenticating with ssh key +1 +Altering credentials and keeping only bcrypt_password +Asserting SSH does not work anymore +AUTHENTICATION_FAILED +Asserting bcrypt_password works +1 +Adding new bcrypt_password +Both current authentication methods should work +1 +1 +Reset authentication methods to new +Only the latest should work, below should fail +AUTHENTICATION_FAILED +Should work +1 +Multiple identified with, not allowed +SYNTAX_ERROR +Multiple identified with, not allowed, even if mixed +SYNTAX_ERROR +Identified with must precede all add identified with, not allowed +SYNTAX_ERROR +CREATE Multiple identified with, not allowed +SYNTAX_ERROR +CREATE Multiple identified with, not allowed, even if mixed +SYNTAX_ERROR +CREATE Identified with must precede all add identified with, not allowed +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh new file mode 100755 index 00000000000..f259f236c96 --- /dev/null +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +function test_login_pwd +{ + ${CLICKHOUSE_CLIENT} --user $1 --password $2 --query "select 1" +} + +function test_login_pwd_expect_error +{ + test_login_pwd "$1" "$2" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' +} + +user="u01_03174" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY '1'" + +echo "Basic authentication after user creation" +test_login_pwd ${user} '1' + +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password BY '2'" + +echo "Changed password, old password should not work" +test_login_pwd_expect_error ${user} '1' + +echo "New password should work" +test_login_pwd ${user} '2' + +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH sha256_password BY '3' ADD IDENTIFIED WITH plaintext_password BY '4'" + +echo "Two new passwords were added, should both work" +test_login_pwd ${user} '3' + +test_login_pwd ${user} '4' + +echo "Gen ssh key" +yes 'yes' | ssh-keygen -t ed25519 -N '' -f 'ssh_key' > /dev/null + +pub_key=$(awk '{print $2}' ssh_key.pub) + +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH ssh_key BY KEY '${pub_key}' TYPE 'ssh-ed25519'" + +echo "Authenticating with ssh key" +${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" + +echo "Altering credentials and keeping only bcrypt_password" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH bcrypt_password BY '5'" + +echo "Asserting SSH does not work anymore" +${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' + +echo "Asserting bcrypt_password works" +test_login_pwd ${user} '5' + +echo "Adding new bcrypt_password" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH bcrypt_password BY '6'" + +echo "Both current authentication methods should work" +test_login_pwd ${user} '5' +test_login_pwd ${user} '6' + +echo "Reset authentication methods to new" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} RESET AUTHENTICATION METHODS TO NEW" + +echo "Only the latest should work, below should fail" +test_login_pwd_expect_error ${user} '5' + +echo "Should work" +test_login_pwd ${user} '6' + +echo "Multiple identified with, not allowed" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +echo "Multiple identified with, not allowed, even if mixed" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password '7' ADD IDENTIFIED WITH plaintext_password '8' IDENTIFIED WITH plaintext_password '9'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +echo "Identified with must precede all add identified with, not allowed" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" + +echo "CREATE Multiple identified with, not allowed" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +echo "CREATE Multiple identified with, not allowed, even if mixed" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password '7' ADD IDENTIFIED WITH plaintext_password '8' IDENTIFIED WITH plaintext_password '9'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +echo "CREATE Identified with must precede all add identified with, not allowed" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From d4da0a0a21e6ead8591848d08b3680b74cdd4629 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 16:30:08 -0300 Subject: [PATCH 014/148] use plaintext_password instead of sha256 because of cicd --- tests/queries/0_stateless/03174_add_identified_with.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index f259f236c96..902002e11e5 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -31,7 +31,7 @@ test_login_pwd_expect_error ${user} '1' echo "New password should work" test_login_pwd ${user} '2' -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH sha256_password BY '3' ADD IDENTIFIED WITH plaintext_password BY '4'" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password BY '3' ADD IDENTIFIED WITH plaintext_password BY '4'" echo "Two new passwords were added, should both work" test_login_pwd ${user} '3' From 5bdd49f36cbd236c97e6cf90c0e424e9689a3859 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 16:43:19 -0300 Subject: [PATCH 015/148] do not gen at runtime, use pre-built ones --- .../03174_add_identified_with.reference | 1 - .../0_stateless/03174_add_identified_with.sh | 15 +++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index bd855d06ffc..c2963686d18 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -7,7 +7,6 @@ New password should work Two new passwords were added, should both work 1 1 -Gen ssh key Authenticating with ssh key 1 Altering credentials and keeping only bcrypt_password diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index 902002e11e5..f13d9e0caf5 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -4,6 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +ssh_key="-----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGAAAAJApc2djKXNn + YwAAAAtzc2gtZWQyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGA + AAAEAk15S5L7j85LvmAivo2J8lo44OR/tLILBO1Wb2//mFwBzqa3duS0ce6QYkzUgko9W0 + Ux7i7d3xPoseFrwnhY4YAAAADWFydGh1ckBhcnRodXI= + -----END OPENSSH PRIVATE KEY-----" + function test_login_pwd { ${CLICKHOUSE_CLIENT} --user $1 --password $2 --query "select 1" @@ -38,12 +46,11 @@ test_login_pwd ${user} '3' test_login_pwd ${user} '4' -echo "Gen ssh key" -yes 'yes' | ssh-keygen -t ed25519 -N '' -f 'ssh_key' > /dev/null +ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" -pub_key=$(awk '{print $2}' ssh_key.pub) +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH ssh_key BY KEY '${pub_key}' TYPE 'ssh-ed25519'" +echo ${ssh_key} > ssh_key echo "Authenticating with ssh key" ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" From 4d5676f4551310380d7cad6e012bb155a2843524 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 16:50:23 -0300 Subject: [PATCH 016/148] dont test ssh at all, it wont work if openssl is not built --- .../03174_add_identified_with.reference | 4 ---- .../0_stateless/03174_add_identified_with.sh | 20 ------------------- 2 files changed, 24 deletions(-) diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index c2963686d18..3d74469d535 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -7,11 +7,7 @@ New password should work Two new passwords were added, should both work 1 1 -Authenticating with ssh key -1 Altering credentials and keeping only bcrypt_password -Asserting SSH does not work anymore -AUTHENTICATION_FAILED Asserting bcrypt_password works 1 Adding new bcrypt_password diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index f13d9e0caf5..2eb71f3a982 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -4,14 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -ssh_key="-----BEGIN OPENSSH PRIVATE KEY----- - b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW - QyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGAAAAJApc2djKXNn - YwAAAAtzc2gtZWQyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGA - AAAEAk15S5L7j85LvmAivo2J8lo44OR/tLILBO1Wb2//mFwBzqa3duS0ce6QYkzUgko9W0 - Ux7i7d3xPoseFrwnhY4YAAAADWFydGh1ckBhcnRodXI= - -----END OPENSSH PRIVATE KEY-----" - function test_login_pwd { ${CLICKHOUSE_CLIENT} --user $1 --password $2 --query "select 1" @@ -46,21 +38,9 @@ test_login_pwd ${user} '3' test_login_pwd ${user} '4' -ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" - -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" - -echo ${ssh_key} > ssh_key - -echo "Authenticating with ssh key" -${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" - echo "Altering credentials and keeping only bcrypt_password" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH bcrypt_password BY '5'" -echo "Asserting SSH does not work anymore" -${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' - echo "Asserting bcrypt_password works" test_login_pwd ${user} '5' From 96cb9f13dd1b53982cab61b245ee25e4fc91d202 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 16:51:34 -0300 Subject: [PATCH 017/148] remove comments --- src/Access/UsersConfigAccessStorage.cpp | 1 - src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index cfb02b0962b..a4fb0972220 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -261,7 +261,6 @@ namespace } else { - // remember to clean it up.. do I need to? smth in my memory tells me I don't need to user->authentication_methods.emplace_back(); } diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index bc507b5f156..a1f8eaa4cb0 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -60,7 +60,6 @@ namespace if (user.authentication_methods.empty() && auth_data.empty()) { - // previously, a user always had a default constructed auth method.. maybe I should put this somewhere else user.authentication_methods.emplace_back(); } From c7aed3c98c2820468e23a8a86e8ca3301dd72254 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 19:37:46 -0300 Subject: [PATCH 018/148] Revert "dont test ssh at all, it wont work if openssl is not built" This reverts commit 4d5676f4551310380d7cad6e012bb155a2843524. --- .../03174_add_identified_with.reference | 4 ++++ .../0_stateless/03174_add_identified_with.sh | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index 3d74469d535..c2963686d18 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -7,7 +7,11 @@ New password should work Two new passwords were added, should both work 1 1 +Authenticating with ssh key +1 Altering credentials and keeping only bcrypt_password +Asserting SSH does not work anymore +AUTHENTICATION_FAILED Asserting bcrypt_password works 1 Adding new bcrypt_password diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index 2eb71f3a982..f13d9e0caf5 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -4,6 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +ssh_key="-----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGAAAAJApc2djKXNn + YwAAAAtzc2gtZWQyNTUxOQAAACAc6mt3bktHHukGJM1IJKPVtFMe4u3d8T6LHha8J4WOGA + AAAEAk15S5L7j85LvmAivo2J8lo44OR/tLILBO1Wb2//mFwBzqa3duS0ce6QYkzUgko9W0 + Ux7i7d3xPoseFrwnhY4YAAAADWFydGh1ckBhcnRodXI= + -----END OPENSSH PRIVATE KEY-----" + function test_login_pwd { ${CLICKHOUSE_CLIENT} --user $1 --password $2 --query "select 1" @@ -38,9 +46,21 @@ test_login_pwd ${user} '3' test_login_pwd ${user} '4' +ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" + +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" + +echo ${ssh_key} > ssh_key + +echo "Authenticating with ssh key" +${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" + echo "Altering credentials and keeping only bcrypt_password" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH bcrypt_password BY '5'" +echo "Asserting SSH does not work anymore" +${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' + echo "Asserting bcrypt_password works" test_login_pwd ${user} '5' From a1928bd2994cd6572fa4a6509216737e8999b819 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 19:39:26 -0300 Subject: [PATCH 019/148] no-fast test --- tests/queries/0_stateless/03174_add_identified_with.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index f13d9e0caf5..060bcd3363a 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 907a54e9f692d0dd352e13f643d29a464190fd67 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 24 Jun 2024 21:21:03 -0300 Subject: [PATCH 020/148] add no parallel --- tests/queries/0_stateless/03174_add_identified_with.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index 060bcd3363a..5b80894d056 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-parallel CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 57afc5f0353ce7b7e6251940f2d17c40b1480855 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Jun 2024 20:02:19 -0300 Subject: [PATCH 021/148] serialize no_password --- src/Access/AuthenticationData.cpp | 3 ++- .../Access/InterpreterShowCreateAccessEntityQuery.cpp | 5 +---- src/Parsers/Access/ASTAuthenticationData.cpp | 2 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 11 +++++++++-- src/Parsers/tests/gtest_Parser.cpp | 8 ++++++++ 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index 70355fadfbd..7191c593eea 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -356,7 +356,8 @@ std::shared_ptr AuthenticationData::toAST() const break; } - case AuthenticationType::NO_PASSWORD: [[fallthrough]]; + case AuthenticationType::NO_PASSWORD: + break; case AuthenticationType::MAX: throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type)); } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 70dd859c70b..e49cf68727c 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -66,10 +66,7 @@ namespace for (const auto & authentication_method : user.authentication_methods) { - if (authentication_method.getType() != AuthenticationType::NO_PASSWORD) - { - query->auth_data.push_back(authentication_method.toAST()); - } + query->auth_data.push_back(authentication_method.toAST()); } if (user.valid_until) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 73fa5a5a692..8dd66808126 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -44,7 +44,7 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt { if (type && *type == AuthenticationType::NO_PASSWORD) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED" + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH no_password" << (settings.hilite ? IAST::hilite_none : ""); return; } diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 0d5db681a7e..86b91fcb6f4 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -189,8 +189,15 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); - // this is weird. - if (!auth_data.empty()) + if (auth_data.empty()) + { + auto ast = std::make_shared(); + ast->type = AuthenticationType::NO_PASSWORD; + + res->auth_data.push_back(ast); + res->children.push_back(ast); + } + else { for (const auto & authentication_method : auth_data) { diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 3b4cd9af695..b88c8f56029 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -284,6 +284,14 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'", "CREATE USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, + { + "CREATE USER user1 IDENTIFIED WITH no_password", + "CREATE USER user1 IDENTIFIED WITH no_password" + }, + { + "CREATE USER user1", + "CREATE USER user1 IDENTIFIED WITH no_password" + }, { "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'", "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'" From 35214a34ee60ca7ee44895d1c0eb57a0e43f0b75 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Jun 2024 22:39:30 -0300 Subject: [PATCH 022/148] update some tests --- tests/queries/0_stateless/01073_grant_and_revoke.reference | 2 +- .../0_stateless/01939_user_with_default_database.reference | 4 ++-- tests/queries/0_stateless/01999_grant_with_replace.reference | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.reference b/tests/queries/0_stateless/01073_grant_and_revoke.reference index b91820914e6..b2c7797ad3c 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.reference +++ b/tests/queries/0_stateless/01073_grant_and_revoke.reference @@ -1,5 +1,5 @@ A -CREATE USER test_user_01073 +CREATE USER test_user_01073 IDENTIFIED WITH no_password B C GRANT INSERT, ALTER DELETE ON *.* TO test_user_01073 diff --git a/tests/queries/0_stateless/01939_user_with_default_database.reference b/tests/queries/0_stateless/01939_user_with_default_database.reference index 8c8ff7e3007..6e5a1d20758 100644 --- a/tests/queries/0_stateless/01939_user_with_default_database.reference +++ b/tests/queries/0_stateless/01939_user_with_default_database.reference @@ -1,4 +1,4 @@ default db_01939 -CREATE USER u_01939 -CREATE USER u_01939 DEFAULT DATABASE NONE +CREATE USER u_01939 IDENTIFIED WITH no_password +CREATE USER u_01939 IDENTIFIED WITH no_password DEFAULT DATABASE NONE diff --git a/tests/queries/0_stateless/01999_grant_with_replace.reference b/tests/queries/0_stateless/01999_grant_with_replace.reference index dc2047ab73c..903f2c301a0 100644 --- a/tests/queries/0_stateless/01999_grant_with_replace.reference +++ b/tests/queries/0_stateless/01999_grant_with_replace.reference @@ -1,4 +1,4 @@ -CREATE USER test_user_01999 +CREATE USER test_user_01999 IDENTIFIED WITH no_password A B GRANT SELECT ON db1.* TO test_user_01999 From 11e537cb282e1ca7b0d65ccaff332afd616d1fbb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 08:51:30 -0300 Subject: [PATCH 023/148] update some more tests --- .../01075_allowed_client_hosts.reference | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.reference b/tests/queries/0_stateless/01075_allowed_client_hosts.reference index 5fb11bae65e..5d2168d6c3b 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.reference +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.reference @@ -1,17 +1,17 @@ -CREATE USER test_user_01075 -CREATE USER test_user_01075 -CREATE USER test_user_01075 HOST NONE -CREATE USER test_user_01075 HOST LOCAL -CREATE USER test_user_01075 HOST IP \'192.168.23.15\' -CREATE USER test_user_01075 HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' -CREATE USER test_user_01075 HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' -CREATE USER test_user_01075 HOST LOCAL -CREATE USER test_user_01075 HOST NONE -CREATE USER test_user_01075 HOST LIKE \'@.somesite.com\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\.anothersite\\\\.org\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\' -CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\' -CREATE USER `test_user_01075_x@localhost` HOST LOCAL -CREATE USER test_user_01075_x HOST LOCAL -CREATE USER `test_user_01075_x@192.168.23.15` HOST LOCAL +CREATE USER test_user_01075 IDENTIFIED WITH no_password +CREATE USER test_user_01075 IDENTIFIED WITH no_password +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST NONE +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST IP \'192.168.23.15\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST NONE +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST LIKE \'@.somesite.com\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite\\\\.com\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\.anothersite\\\\.org\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\' +CREATE USER test_user_01075 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\' +CREATE USER `test_user_01075_x@localhost` IDENTIFIED WITH no_password HOST LOCAL +CREATE USER test_user_01075_x IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `test_user_01075_x@192.168.23.15` IDENTIFIED WITH no_password HOST LOCAL From 4d800a848741a71e6805050367de77e2edf5f646 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 08:54:58 -0300 Subject: [PATCH 024/148] update test_disk_access_storage --- tests/integration/test_disk_access_storage/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index a710295505e..7414ff4d84b 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -46,7 +46,7 @@ def test_create(): def check(): assert ( instance.query("SHOW CREATE USER u1") - == "CREATE USER u1 SETTINGS PROFILE `s1`\n" + == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS PROFILE `s1`\n" ) assert ( instance.query("SHOW CREATE USER u2") @@ -99,7 +99,7 @@ def test_alter(): def check(): assert ( instance.query("SHOW CREATE USER u1") - == "CREATE USER u1 SETTINGS PROFILE `s1`\n" + == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS PROFILE `s1`\n" ) assert ( instance.query("SHOW CREATE USER u2") @@ -147,7 +147,7 @@ def test_drop(): instance.query("DROP SETTINGS PROFILE s1") def check(): - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1\n" + assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH no_password\n" assert ( instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE `s2`\n" From b93c21a0410b5930b359be0e1ae5683d14978881 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 08:58:44 -0300 Subject: [PATCH 025/148] add no-parallel to grant_and_Revoke.sql --- tests/queries/0_stateless/01073_grant_and_revoke.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01073_grant_and_revoke.sql b/tests/queries/0_stateless/01073_grant_and_revoke.sql index 4cffd916e9f..59f599ce140 100644 --- a/tests/queries/0_stateless/01073_grant_and_revoke.sql +++ b/tests/queries/0_stateless/01073_grant_and_revoke.sql @@ -1,3 +1,5 @@ +-- Tags: no-parallel + DROP USER IF EXISTS test_user_01073; DROP ROLE IF EXISTS test_role_01073; From f15551b47b4cc54eeb1c6ece75c9f5c37f90bb13 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 09:38:55 -0300 Subject: [PATCH 026/148] black --- tests/integration/test_disk_access_storage/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index 7414ff4d84b..ae756ef1e27 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -147,7 +147,10 @@ def test_drop(): instance.query("DROP SETTINGS PROFILE s1") def check(): - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH no_password\n" + assert ( + instance.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH no_password\n" + ) assert ( instance.query("SHOW CREATE SETTINGS PROFILE s2") == "CREATE SETTINGS PROFILE `s2`\n" From 341071402c9d2c1f4f19d0f98cba8de1d3b25d90 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 11:00:13 -0300 Subject: [PATCH 027/148] fix wrong expected error code & add test --- .../03174_add_identified_with.reference | 15 ++++++++----- .../0_stateless/03174_add_identified_with.sh | 22 ++++++++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index c2963686d18..e8e1ee9d294 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -24,14 +24,17 @@ AUTHENTICATION_FAILED Should work 1 Multiple identified with, not allowed -SYNTAX_ERROR +BAD_ARGUMENTS Multiple identified with, not allowed, even if mixed -SYNTAX_ERROR +BAD_ARGUMENTS Identified with must precede all add identified with, not allowed -SYNTAX_ERROR +BAD_ARGUMENTS CREATE Multiple identified with, not allowed -SYNTAX_ERROR +BAD_ARGUMENTS CREATE Multiple identified with, not allowed, even if mixed -SYNTAX_ERROR +BAD_ARGUMENTS CREATE Identified with must precede all add identified with, not allowed -SYNTAX_ERROR +BAD_ARGUMENTS +Create user with no identification +Add identified with +CREATE USER u01_03174 IDENTIFIED WITH no_password ADD IDENTIFIED WITH plaintext_password diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index 5b80894d056..0884f31dd73 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -82,19 +82,29 @@ echo "Should work" test_login_pwd ${user} '6' echo "Multiple identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" echo "Multiple identified with, not allowed, even if mixed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password '7' ADD IDENTIFIED WITH plaintext_password '8' IDENTIFIED WITH plaintext_password '9'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password by '7' ADD IDENTIFIED WITH plaintext_password by '8' IDENTIFIED WITH plaintext_password by '9'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" echo "Identified with must precede all add identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" ${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" echo "CREATE Multiple identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" echo "CREATE Multiple identified with, not allowed, even if mixed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password '7' ADD IDENTIFIED WITH plaintext_password '8' IDENTIFIED WITH plaintext_password '9'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '7' ADD IDENTIFIED WITH plaintext_password by '8' IDENTIFIED WITH plaintext_password by '9'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" echo "CREATE Identified with must precede all add identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password '7' IDENTIFIED WITH plaintext_password '8'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +echo "Create user with no identification" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user}" + +echo "Add identified with" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '7'" + +${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 3e77101b164eb3855f377a203e511186e9afb908 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 13:29:13 -0300 Subject: [PATCH 028/148] update some other tests --- tests/integration/test_access_control_on_cluster/test.py | 6 +++--- tests/queries/0_stateless/01075_allowed_client_hosts.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index b12add7ad3f..3933ab15e0e 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -42,9 +42,9 @@ def test_access_control_on_cluster(): ch1.query_with_retry( "CREATE USER IF NOT EXISTS Alex ON CLUSTER 'cluster'", retry_count=5 ) - assert ch1.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" - assert ch2.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" - assert ch3.query("SHOW CREATE USER Alex") == "CREATE USER Alex\n" + assert ch2.query("SHOW CREATE USER Alex") == "CREATE USER Alex IDENTIFIED WITH no_password\n" + assert ch1.query("SHOW CREATE USER Alex") == "CREATE USER Alex IDENTIFIED WITH no_password\n" + assert ch3.query("SHOW CREATE USER Alex") == "CREATE USER Alex IDENTIFIED WITH no_password\n" ch2.query_with_retry( "GRANT ON CLUSTER 'cluster' SELECT ON *.* TO Alex", retry_count=3 diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.sql b/tests/queries/0_stateless/01075_allowed_client_hosts.sql index 17957c8826b..8c25d45f421 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.sql +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-parallel DROP USER IF EXISTS test_user_01075, test_user_01075_x, test_user_01075_x@localhost, test_user_01075_x@'192.168.23.15'; From a77a0b5eb011e4b4907b903c347f4a85c461b984 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 13:59:57 -0300 Subject: [PATCH 029/148] black --- .../test_access_control_on_cluster/test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index 3933ab15e0e..a754292bfff 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -42,9 +42,18 @@ def test_access_control_on_cluster(): ch1.query_with_retry( "CREATE USER IF NOT EXISTS Alex ON CLUSTER 'cluster'", retry_count=5 ) - assert ch2.query("SHOW CREATE USER Alex") == "CREATE USER Alex IDENTIFIED WITH no_password\n" - assert ch1.query("SHOW CREATE USER Alex") == "CREATE USER Alex IDENTIFIED WITH no_password\n" - assert ch3.query("SHOW CREATE USER Alex") == "CREATE USER Alex IDENTIFIED WITH no_password\n" + assert ( + ch2.query("SHOW CREATE USER Alex") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" + ) + assert ( + ch1.query("SHOW CREATE USER Alex") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" + ) + assert ( + ch3.query("SHOW CREATE USER Alex") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" + ) ch2.query_with_retry( "GRANT ON CLUSTER 'cluster' SELECT ON *.* TO Alex", retry_count=3 From 938004c090b112f4a76c904bcf26f4f6bed5a867 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 16:40:40 -0300 Subject: [PATCH 030/148] update yet another test --- .../0_stateless/01292_create_user.reference | 158 +++++++++--------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index d5841a74a2c..c23723b3ec1 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -1,12 +1,12 @@ -- default -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password -- same as default -CREATE USER u2_01292 -CREATE USER u3_01292 +CREATE USER u2_01292 IDENTIFIED WITH no_password +CREATE USER u3_01292 IDENTIFIED WITH no_password -- rename -CREATE USER u2_01292_renamed +CREATE USER u2_01292_renamed IDENTIFIED WITH no_password -- authentication -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password CREATE USER u2_01292 IDENTIFIED WITH plaintext_password CREATE USER u3_01292 IDENTIFIED WITH sha256_password CREATE USER u4_01292 IDENTIFIED WITH sha256_password @@ -19,92 +19,92 @@ CREATE USER u1_01292 IDENTIFIED WITH sha256_password CREATE USER u2_01292 IDENTIFIED WITH sha256_password CREATE USER u3_01292 IDENTIFIED WITH sha256_password CREATE USER u4_01292 IDENTIFIED WITH plaintext_password -CREATE USER u5_01292 +CREATE USER u5_01292 IDENTIFIED WITH no_password -- host -CREATE USER u1_01292 -CREATE USER u2_01292 HOST NONE -CREATE USER u3_01292 HOST LOCAL -CREATE USER u4_01292 HOST NAME \'myhost.com\' -CREATE USER u5_01292 HOST LOCAL, NAME \'myhost.com\' -CREATE USER u6_01292 HOST LOCAL, NAME \'myhost.com\' -CREATE USER u7_01292 HOST REGEXP \'.*\\\\.myhost\\\\.com\' -CREATE USER u8_01292 -CREATE USER u9_01292 HOST LIKE \'%.myhost.com\' -CREATE USER u10_01292 HOST LIKE \'%.myhost.com\' -CREATE USER u11_01292 HOST LOCAL -CREATE USER u12_01292 HOST IP \'192.168.1.1\' -CREATE USER u13_01292 HOST IP \'192.168.0.0/16\' -CREATE USER u14_01292 HOST LOCAL -CREATE USER u15_01292 HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' -CREATE USER u16_01292 HOST LOCAL, IP \'65:ff0c::/96\' -CREATE USER u1_01292 HOST NONE -CREATE USER u2_01292 HOST NAME \'myhost.com\' -CREATE USER u3_01292 HOST LOCAL, NAME \'myhost.com\' -CREATE USER u4_01292 HOST NONE +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password HOST NONE +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST NAME \'myhost.com\' +CREATE USER u5_01292 IDENTIFIED WITH no_password HOST LOCAL, NAME \'myhost.com\' +CREATE USER u6_01292 IDENTIFIED WITH no_password HOST LOCAL, NAME \'myhost.com\' +CREATE USER u7_01292 IDENTIFIED WITH no_password HOST REGEXP \'.*\\\\.myhost\\\\.com\' +CREATE USER u8_01292 IDENTIFIED WITH no_password +CREATE USER u9_01292 IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER u10_01292 IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER u11_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER u12_01292 IDENTIFIED WITH no_password HOST IP \'192.168.1.1\' +CREATE USER u13_01292 IDENTIFIED WITH no_password HOST IP \'192.168.0.0/16\' +CREATE USER u14_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER u15_01292 IDENTIFIED WITH no_password HOST IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765d\' +CREATE USER u16_01292 IDENTIFIED WITH no_password HOST LOCAL, IP \'65:ff0c::/96\' +CREATE USER u1_01292 IDENTIFIED WITH no_password HOST NONE +CREATE USER u2_01292 IDENTIFIED WITH no_password HOST NAME \'myhost.com\' +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LOCAL, NAME \'myhost.com\' +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST NONE -- host after @ -CREATE USER u1_01292 -CREATE USER u1_01292 -CREATE USER `u2_01292@%.myhost.com` HOST LIKE \'%.myhost.com\' -CREATE USER `u2_01292@%.myhost.com` HOST LIKE \'%.myhost.com\' -CREATE USER `u3_01292@192.168.%.%` HOST LIKE \'192.168.%.%\' -CREATE USER `u3_01292@192.168.%.%` HOST LIKE \'192.168.%.%\' -CREATE USER `u4_01292@::1` HOST LOCAL -CREATE USER `u4_01292@::1` HOST LOCAL -CREATE USER `u5_01292@65:ff0c::/96` HOST LIKE \'65:ff0c::/96\' -CREATE USER `u5_01292@65:ff0c::/96` HOST LIKE \'65:ff0c::/96\' -CREATE USER u1_01292 HOST LOCAL -CREATE USER `u2_01292@%.myhost.com` +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER `u2_01292@%.myhost.com` IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER `u2_01292@%.myhost.com` IDENTIFIED WITH no_password HOST LIKE \'%.myhost.com\' +CREATE USER `u3_01292@192.168.%.%` IDENTIFIED WITH no_password HOST LIKE \'192.168.%.%\' +CREATE USER `u3_01292@192.168.%.%` IDENTIFIED WITH no_password HOST LIKE \'192.168.%.%\' +CREATE USER `u4_01292@::1` IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `u4_01292@::1` IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `u5_01292@65:ff0c::/96` IDENTIFIED WITH no_password HOST LIKE \'65:ff0c::/96\' +CREATE USER `u5_01292@65:ff0c::/96` IDENTIFIED WITH no_password HOST LIKE \'65:ff0c::/96\' +CREATE USER u1_01292 IDENTIFIED WITH no_password HOST LOCAL +CREATE USER `u2_01292@%.myhost.com` IDENTIFIED WITH no_password -- settings -CREATE USER u1_01292 -CREATE USER u2_01292 SETTINGS PROFILE `default` -CREATE USER u3_01292 SETTINGS max_memory_usage = 5000000 -CREATE USER u4_01292 SETTINGS max_memory_usage MIN 5000000 -CREATE USER u5_01292 SETTINGS max_memory_usage MAX 5000000 -CREATE USER u6_01292 SETTINGS max_memory_usage CONST -CREATE USER u7_01292 SETTINGS max_memory_usage WRITABLE -CREATE USER u8_01292 SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST -CREATE USER u9_01292 SETTINGS PROFILE `default`, max_memory_usage = 5000000 WRITABLE -CREATE USER u1_01292 SETTINGS readonly = 1 -CREATE USER u2_01292 SETTINGS readonly = 1 -CREATE USER u3_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password SETTINGS PROFILE `default` +CREATE USER u3_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage = 5000000 +CREATE USER u4_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage MIN 5000000 +CREATE USER u5_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage MAX 5000000 +CREATE USER u6_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage CONST +CREATE USER u7_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage WRITABLE +CREATE USER u8_01292 IDENTIFIED WITH no_password SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST +CREATE USER u9_01292 IDENTIFIED WITH no_password SETTINGS PROFILE `default`, max_memory_usage = 5000000 WRITABLE +CREATE USER u1_01292 IDENTIFIED WITH no_password SETTINGS readonly = 1 +CREATE USER u2_01292 IDENTIFIED WITH no_password SETTINGS readonly = 1 +CREATE USER u3_01292 IDENTIFIED WITH no_password -- default role -CREATE USER u1_01292 -CREATE USER u2_01292 DEFAULT ROLE NONE -CREATE USER u3_01292 DEFAULT ROLE r1_01292 -CREATE USER u4_01292 DEFAULT ROLE r1_01292, r2_01292 -CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r2_01292 -CREATE USER u6_01292 DEFAULT ROLE ALL EXCEPT r1_01292, r2_01292 -CREATE USER u1_01292 DEFAULT ROLE r1_01292 -CREATE USER u2_01292 DEFAULT ROLE ALL EXCEPT r2_01292 -CREATE USER u3_01292 DEFAULT ROLE r2_01292 -CREATE USER u4_01292 -CREATE USER u5_01292 DEFAULT ROLE ALL EXCEPT r1_01292 -CREATE USER u6_01292 DEFAULT ROLE NONE +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE +CREATE USER u3_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292 +CREATE USER u4_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u5_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r2_01292 +CREATE USER u6_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r1_01292, r2_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292 +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r2_01292 +CREATE USER u3_01292 IDENTIFIED WITH no_password DEFAULT ROLE r2_01292 +CREATE USER u4_01292 IDENTIFIED WITH no_password +CREATE USER u5_01292 IDENTIFIED WITH no_password DEFAULT ROLE ALL EXCEPT r1_01292 +CREATE USER u6_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE -- complex CREATE USER u1_01292 IDENTIFIED WITH plaintext_password HOST LOCAL SETTINGS readonly = 1 -CREATE USER u1_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE NONE SETTINGS PROFILE `default` +CREATE USER u1_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE NONE SETTINGS PROFILE `default` -- if not exists -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password GRANT r1_01292 TO u1_01292 -- if not exists-part2 -CREATE USER u1_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password GRANT r1_01292, r2_01292 TO u1_01292 -- or replace -CREATE USER u1_01292 -CREATE USER u2_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password +CREATE USER u2_01292 IDENTIFIED WITH no_password -- multiple users in one command -CREATE USER u1_01292 DEFAULT ROLE NONE -CREATE USER u2_01292 DEFAULT ROLE NONE -CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' -CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' -CREATE USER `u5_01292@%.host.com` HOST LIKE \'%.host.com\' -CREATE USER `u6_01292@%.host.com` HOST LIKE \'%.host.com\' -CREATE USER `u7_01292@%.host.com` HOST LIKE \'%.host.com\' -CREATE USER `u8_01292@%.otherhost.com` HOST LIKE \'%.otherhost.com\' -CREATE USER u1_01292 DEFAULT ROLE NONE SETTINGS readonly = 1 -CREATE USER u2_01292 DEFAULT ROLE r1_01292, r2_01292 SETTINGS readonly = 1 -CREATE USER u3_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 -CREATE USER u4_01292 HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u1_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' +CREATE USER `u5_01292@%.host.com` IDENTIFIED WITH no_password HOST LIKE \'%.host.com\' +CREATE USER `u6_01292@%.host.com` IDENTIFIED WITH no_password HOST LIKE \'%.host.com\' +CREATE USER `u7_01292@%.host.com` IDENTIFIED WITH no_password HOST LIKE \'%.host.com\' +CREATE USER `u8_01292@%.otherhost.com` IDENTIFIED WITH no_password HOST LIKE \'%.otherhost.com\' +CREATE USER u1_01292 IDENTIFIED WITH no_password DEFAULT ROLE NONE SETTINGS readonly = 1 +CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292, r2_01292 SETTINGS readonly = 1 +CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 +CREATE USER u4_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 -- system.users u1_01292 local_directory plaintext_password {} [] ['localhost'] [] [] 1 [] [] u2_01292 local_directory no_password {} [] [] [] ['%.%.myhost.com'] 0 [] [] From c0e1095e66670289cc83d8d104bd508412029a85 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 17:06:38 -0300 Subject: [PATCH 031/148] initial fix for session_log, shall be refactored --- src/Access/Authentication.cpp | 23 +---------------------- src/Access/IAccessStorage.cpp | 27 +++++++++++++++------------ src/Access/IAccessStorage.h | 6 +++++- src/Access/LDAPAccessStorage.cpp | 2 +- src/Interpreters/Session.cpp | 4 +++- src/Interpreters/Session.h | 1 + src/Interpreters/SessionLog.cpp | 11 ++++++----- src/Interpreters/SessionLog.h | 5 +++-- 8 files changed, 35 insertions(+), 44 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 827f2e17e94..7174c66d4b1 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -249,27 +249,6 @@ namespace return false; } #endif - - [[noreturn]] void throwInvalidCredentialsException(const std::vector & authentication_methods) - { - std::string possible_authentication_types; - bool first = true; - - for (const auto & authentication_method : authentication_methods) - { - if (!first) - { - possible_authentication_types += ", "; - } - possible_authentication_types += toString(authentication_method.getType()); - first = false; - } - - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "areCredentialsValid(): Invalid credentials provided, available authentication methods are {}", - possible_authentication_types); - } } bool Authentication::areCredentialsValid( @@ -311,7 +290,7 @@ bool Authentication::areCredentialsValid( if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast(&credentials)) return true; - throwInvalidCredentialsException(authentication_methods); + return false; } } diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index f1725dafdf4..5b29c07e046 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -521,7 +521,7 @@ std::optional IAccessStorage::authenticateImpl( { if (auto user = tryRead(*id)) { - AuthResult auth_result { .user_id = *id }; + AuthResult auth_result { .user_id = *id, .authentication_data = std::nullopt }; if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); @@ -534,12 +534,15 @@ std::optional IAccessStorage::authenticateImpl( if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) throwAuthenticationTypeNotAllowed(auth_type); + + if (areCredentialsValid(user->getName(), user->valid_until, auth_method, credentials, external_authenticators, auth_result.settings)) + { + auth_result.authentication_data = auth_method; + return auth_result; + } } - if (!areCredentialsValid(*user, credentials, external_authenticators, auth_result.settings)) - throwInvalidCredentials(); - - return auth_result; + throwInvalidCredentials(); } } @@ -549,9 +552,10 @@ std::optional IAccessStorage::authenticateImpl( return std::nullopt; } - bool IAccessStorage::areCredentialsValid( - const User & user, + const std::string user_name, + time_t valid_until, + const AuthenticationData & authentication_method, const Credentials & credentials, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) const @@ -559,21 +563,20 @@ bool IAccessStorage::areCredentialsValid( if (!credentials.isReady()) return false; - if (credentials.getUserName() != user.getName()) + if (credentials.getUserName() != user_name) return false; - if (user.valid_until) + if (valid_until) { const time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - if (now > user.valid_until) + if (now > valid_until) return false; } - return Authentication::areCredentialsValid(credentials, user.authentication_methods, external_authenticators, settings); + return Authentication::areCredentialsValid(credentials, {authentication_method}, external_authenticators, settings); } - bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const { return user.allowed_client_hosts.contains(address); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index e88b1601f32..df2e8739f7a 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -34,6 +35,7 @@ struct AuthResult UUID user_id; /// Session settings received from authentication server (if any) SettingsChanges settings{}; + std::optional authentication_data; }; /// Contains entities, i.e. instances of classes derived from IAccessEntity. @@ -227,7 +229,9 @@ protected: bool allow_no_password, bool allow_plaintext_password) const; virtual bool areCredentialsValid( - const User & user, + const std::string user_name, + time_t valid_until, + const AuthenticationData & authentication_method, const Credentials & credentials, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) const; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 89920878fef..f6435b0d899 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -504,7 +504,7 @@ std::optional LDAPAccessStorage::authenticateImpl( } if (id) - return AuthResult{ .user_id = *id }; + return AuthResult{ .user_id = *id, .authentication_data = AuthenticationData(AuthenticationType::LDAP) }; return std::nullopt; } diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 8bacc3ac8fc..f56fcb2172f 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -361,6 +361,7 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So { auto auth_result = global_context->getAccessControl().authenticate(credentials_, address.host(), getClientInfo().getLastForwardedFor()); user_id = auth_result.user_id; + user_authenticated_with = auth_result.authentication_data; settings_from_auth_server = auth_result.settings; LOG_DEBUG(log, "{} Authenticated with global context as user {}", toString(auth_id), toString(*user_id)); @@ -705,7 +706,8 @@ void Session::recordLoginSuccess(ContextPtr login_context) const settings, access->getAccess(), getClientInfo(), - user); + user, + *user_authenticated_with); } notified_session_log_about_login = true; diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index f671969b1d4..6281df25de3 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -113,6 +113,7 @@ private: mutable UserPtr user; std::optional user_id; + std::optional user_authenticated_with; ContextMutablePtr session_context; mutable bool query_context_created = false; diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index 4832450af48..30c3bd570f3 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -208,13 +208,13 @@ void SessionLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insertData(auth_failure_reason.data(), auth_failure_reason.length()); } -// todo arthur fix this method void SessionLog::addLoginSuccess(const UUID & auth_id, const String & session_id, const Settings & settings, const ContextAccessPtr & access, const ClientInfo & client_info, - const UserPtr & login_user) + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with) { SessionLogElement log_entry(auth_id, SESSION_LOGIN_SUCCESS); log_entry.client_info = client_info; @@ -222,10 +222,11 @@ void SessionLog::addLoginSuccess(const UUID & auth_id, if (login_user) { log_entry.user = login_user->getName(); - log_entry.user_identified_with = login_user->authentication_methods.back().getType(); -// log_entry.user_identified_with = login_user->auth_data.getType(); + log_entry.user_identified_with = user_authenticated_with.getType(); } - log_entry.external_auth_server = login_user ? login_user->authentication_methods.back().getLDAPServerName() : ""; + + log_entry.external_auth_server = user_authenticated_with.getLDAPServerName(); + log_entry.session_id = session_id; diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h index 5bacb9677c0..2037d6c6861 100644 --- a/src/Interpreters/SessionLog.h +++ b/src/Interpreters/SessionLog.h @@ -22,6 +22,7 @@ class ContextAccess; struct User; using UserPtr = std::shared_ptr; using ContextAccessPtr = std::shared_ptr; +class AuthenticationData; /** A struct which will be inserted as row into session_log table. * @@ -71,14 +72,14 @@ struct SessionLogElement class SessionLog : public SystemLog { using SystemLog::SystemLog; - public: void addLoginSuccess(const UUID & auth_id, const String & session_id, const Settings & settings, const ContextAccessPtr & access, const ClientInfo & client_info, - const UserPtr & login_user); + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with); void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional & user, const Exception & reason); void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info); From 95f584c5d05ddd2f1642456cbedadcabee2f0b23 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 17:25:46 -0300 Subject: [PATCH 032/148] black --- src/Access/Authentication.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 7174c66d4b1..e38bd5abfe5 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -15,7 +15,6 @@ namespace DB { namespace ErrorCodes { - extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; } From 678700f137f0932505a8c6101569b9c6d12b3764 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 26 Jun 2024 22:26:46 -0300 Subject: [PATCH 033/148] update some other tests --- .../integration/test_backup_restore_new/test.py | 2 +- .../test_backup_restore_on_cluster/test.py | 2 +- .../test_enabling_access_management/test.py | 4 ++-- tests/integration/test_grant_and_revoke/test.py | 16 ++++++++-------- tests/integration/test_settings_profile/test.py | 4 ++-- tests/integration/test_user_valid_until/test.py | 14 +++++++------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index d8662fad011..78b906053a7 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1212,7 +1212,7 @@ def test_system_users_required_privileges(): instance.query("GRANT SELECT ON test.* TO u2 WITH GRANT OPTION") instance.query(f"RESTORE ALL FROM {backup_name}", user="u2") - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 DEFAULT ROLE r1\n" + assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH no_password DEFAULT ROLE r1\n" assert instance.query("SHOW GRANTS FOR u1") == TSV( ["GRANT SELECT ON test.* TO u1", "GRANT r1 TO u1"] ) diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 700ed6f15f5..3a15bc1d947 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -769,7 +769,7 @@ def test_system_users(): ) assert ( - node1.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS custom_a = 123\n" + node1.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS custom_a = 123\n" ) assert node1.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON default.tbl TO u1\n" diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index 0b8c1771a40..450ff267a5a 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -20,10 +20,10 @@ def started_cluster(): def test_enabling_access_management(): instance.query("CREATE USER Alex", user="default") assert ( - instance.query("SHOW CREATE USER Alex", user="default") == "CREATE USER Alex\n" + instance.query("SHOW CREATE USER Alex", user="default") == "CREATE USER Alex IDENTIFIED WITH no_password\n" ) assert ( - instance.query("SHOW CREATE USER Alex", user="readonly") == "CREATE USER Alex\n" + instance.query("SHOW CREATE USER Alex", user="readonly") == "CREATE USER Alex IDENTIFIED WITH no_password\n" ) assert "Not enough privileges" in instance.query_and_get_error( "SHOW CREATE USER Alex", user="xyz" diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index e533cced1e4..9748675dccf 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -144,7 +144,7 @@ def test_allowed_grantees(): instance.query("ALTER USER A GRANTEES ANY EXCEPT B") assert ( - instance.query("SHOW CREATE USER A") == "CREATE USER A GRANTEES ANY EXCEPT B\n" + instance.query("SHOW CREATE USER A") == "CREATE USER A IDENTIFIED WITH no_password GRANTEES ANY EXCEPT B\n" ) expected_error = "user `B` is not allowed as grantee" assert expected_error in instance.query_and_get_error( @@ -157,7 +157,7 @@ def test_allowed_grantees(): instance.query("REVOKE SELECT ON test.table FROM B", user="A") instance.query("ALTER USER A GRANTEES ANY") - assert instance.query("SHOW CREATE USER A") == "CREATE USER A\n" + assert instance.query("SHOW CREATE USER A") == "CREATE USER A IDENTIFIED WITH no_password\n" instance.query("GRANT SELECT ON test.table TO B", user="A") assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" @@ -169,7 +169,7 @@ def test_allowed_grantees(): instance.query("CREATE USER C GRANTEES ANY EXCEPT C") assert ( - instance.query("SHOW CREATE USER C") == "CREATE USER C GRANTEES ANY EXCEPT C\n" + instance.query("SHOW CREATE USER C") == "CREATE USER C IDENTIFIED WITH no_password GRANTEES ANY EXCEPT C\n" ) instance.query("GRANT SELECT ON test.table TO C WITH GRANT OPTION") assert instance.query("SELECT * FROM test.table", user="C") == "1\t5\n2\t10\n" @@ -385,15 +385,15 @@ def test_introspection(): instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) - assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"]) - assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"]) + assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A IDENTIFIED WITH no_password"]) + assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B IDENTIFIED WITH no_password"]) assert instance.query("SHOW CREATE USERS A,B") == TSV( - ["CREATE USER A", "CREATE USER B"] + ["CREATE USER A IDENTIFIED WITH no_password", "CREATE USER B IDENTIFIED WITH no_password"] ) assert instance.query("SHOW CREATE USERS") == TSV( [ - "CREATE USER A", - "CREATE USER B", + "CREATE USER A IDENTIFIED WITH no_password", + "CREATE USER B IDENTIFIED WITH no_password", "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE `default`", ] ) diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index e5c0a072ff9..bfd93744832 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -128,7 +128,7 @@ def test_smoke(): instance.query("ALTER USER robin SETTINGS PROFILE xyz") assert ( instance.query("SHOW CREATE USER robin") - == "CREATE USER robin SETTINGS PROFILE `xyz`\n" + == "CREATE USER robin IDENTIFIED WITH no_password SETTINGS PROFILE `xyz`\n" ) assert ( instance.query( @@ -152,7 +152,7 @@ def test_smoke(): ] instance.query("ALTER USER robin SETTINGS NONE") - assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin\n" + assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin IDENTIFIED WITH no_password\n" assert ( instance.query( "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py index d6d5bf8b18e..487a9a2744a 100644 --- a/tests/integration/test_user_valid_until/test.py +++ b/tests/integration/test_user_valid_until/test.py @@ -22,7 +22,7 @@ def test_basic(started_cluster): # 1. Without VALID UNTIL node.query("CREATE USER user_basic") - assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic\n" + assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic IDENTIFIED WITH no_password\n" assert node.query("SELECT 1", user="user_basic") == "1\n" # 2. With valid VALID UNTIL @@ -30,7 +30,7 @@ def test_basic(started_cluster): assert ( node.query("SHOW CREATE USER user_basic") - == "CREATE USER user_basic VALID UNTIL \\'2040-11-06 05:03:20\\'\n" + == "CREATE USER user_basic IDENTIFIED WITH no_password VALID UNTIL \\'2040-11-06 05:03:20\\'\n" ) assert node.query("SELECT 1", user="user_basic") == "1\n" @@ -39,7 +39,7 @@ def test_basic(started_cluster): assert ( node.query("SHOW CREATE USER user_basic") - == "CREATE USER user_basic VALID UNTIL \\'2010-11-06 05:03:20\\'\n" + == "CREATE USER user_basic IDENTIFIED WITH no_password VALID UNTIL \\'2010-11-06 05:03:20\\'\n" ) error = "Authentication failed" @@ -48,7 +48,7 @@ def test_basic(started_cluster): # 4. Reset VALID UNTIL node.query("ALTER USER user_basic VALID UNTIL 'infinity'") - assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic\n" + assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic IDENTIFIED WITH no_password\n" assert node.query("SELECT 1", user="user_basic") == "1\n" node.query("DROP USER user_basic") @@ -72,15 +72,15 @@ def test_details(started_cluster): assert ( node.query("SHOW CREATE USER user_details_infinity") - == "CREATE USER user_details_infinity\n" + == "CREATE USER user_details_infinity IDENTIFIED WITH no_password\n" ) # 2. Time only is not supported - node.query("CREATE USER user_details_time_only VALID UNTIL '22:03:40'") + node.query("CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL '22:03:40'") until_year = datetime.today().strftime("%Y") assert ( node.query("SHOW CREATE USER user_details_time_only") - == f"CREATE USER user_details_time_only VALID UNTIL \\'{until_year}-01-01 22:03:40\\'\n" + == f"CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL \\'{until_year}-01-01 22:03:40\\'\n" ) From ec0d426a6f4c6fe95bf4c185f6880bc26943050c Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 08:52:20 -0300 Subject: [PATCH 034/148] try to fix black --- tests/integration/test_backup_restore_new/test.py | 5 ++++- .../test_backup_restore_on_cluster/test.py | 3 ++- .../test_enabling_access_management/test.py | 6 ++++-- tests/integration/test_grant_and_revoke/test.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 78b906053a7..b330a509e05 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1212,7 +1212,10 @@ def test_system_users_required_privileges(): instance.query("GRANT SELECT ON test.* TO u2 WITH GRANT OPTION") instance.query(f"RESTORE ALL FROM {backup_name}", user="u2") - assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH no_password DEFAULT ROLE r1\n" + assert ( + instance.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH no_password DEFAULT ROLE r1\n" + ) assert instance.query("SHOW GRANTS FOR u1") == TSV( ["GRANT SELECT ON test.* TO u1", "GRANT r1 TO u1"] ) diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 3a15bc1d947..5415820bed6 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -769,7 +769,8 @@ def test_system_users(): ) assert ( - node1.query("SHOW CREATE USER u1") == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS custom_a = 123\n" + node1.query("SHOW CREATE USER u1") + == "CREATE USER u1 IDENTIFIED WITH no_password SETTINGS custom_a = 123\n" ) assert node1.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON default.tbl TO u1\n" diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index 450ff267a5a..821e598d385 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -20,10 +20,12 @@ def started_cluster(): def test_enabling_access_management(): instance.query("CREATE USER Alex", user="default") assert ( - instance.query("SHOW CREATE USER Alex", user="default") == "CREATE USER Alex IDENTIFIED WITH no_password\n" + instance.query("SHOW CREATE USER Alex", user="default") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" ) assert ( - instance.query("SHOW CREATE USER Alex", user="readonly") == "CREATE USER Alex IDENTIFIED WITH no_password\n" + instance.query("SHOW CREATE USER Alex", user="readonly") + == "CREATE USER Alex IDENTIFIED WITH no_password\n" ) assert "Not enough privileges" in instance.query_and_get_error( "SHOW CREATE USER Alex", user="xyz" diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 9748675dccf..a1ce37a5d3d 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -144,7 +144,8 @@ def test_allowed_grantees(): instance.query("ALTER USER A GRANTEES ANY EXCEPT B") assert ( - instance.query("SHOW CREATE USER A") == "CREATE USER A IDENTIFIED WITH no_password GRANTEES ANY EXCEPT B\n" + instance.query("SHOW CREATE USER A") + == "CREATE USER A IDENTIFIED WITH no_password GRANTEES ANY EXCEPT B\n" ) expected_error = "user `B` is not allowed as grantee" assert expected_error in instance.query_and_get_error( @@ -157,7 +158,10 @@ def test_allowed_grantees(): instance.query("REVOKE SELECT ON test.table FROM B", user="A") instance.query("ALTER USER A GRANTEES ANY") - assert instance.query("SHOW CREATE USER A") == "CREATE USER A IDENTIFIED WITH no_password\n" + assert ( + instance.query("SHOW CREATE USER A") + == "CREATE USER A IDENTIFIED WITH no_password\n" + ) instance.query("GRANT SELECT ON test.table TO B", user="A") assert instance.query("SELECT * FROM test.table", user="B") == "1\t5\n2\t10\n" @@ -169,7 +173,8 @@ def test_allowed_grantees(): instance.query("CREATE USER C GRANTEES ANY EXCEPT C") assert ( - instance.query("SHOW CREATE USER C") == "CREATE USER C IDENTIFIED WITH no_password GRANTEES ANY EXCEPT C\n" + instance.query("SHOW CREATE USER C") + == "CREATE USER C IDENTIFIED WITH no_password GRANTEES ANY EXCEPT C\n" ) instance.query("GRANT SELECT ON test.table TO C WITH GRANT OPTION") assert instance.query("SELECT * FROM test.table", user="C") == "1\t5\n2\t10\n" From f69a19d01d18848e08ff2e7b81c064b9da0da160 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 09:20:12 -0300 Subject: [PATCH 035/148] try to fix black --- tests/integration/test_grant_and_revoke/test.py | 15 ++++++++++++--- tests/integration/test_settings_profile/test.py | 5 ++++- tests/integration/test_user_valid_until/test.py | 14 +++++++++++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index a1ce37a5d3d..9fc3dfc249a 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -390,10 +390,19 @@ def test_introspection(): instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) - assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A IDENTIFIED WITH no_password"]) - assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B IDENTIFIED WITH no_password"]) + assert ( + instance.query("SHOW CREATE USERS A") + == TSV(["CREATE USER A IDENTIFIED WITH no_password"]) + ) + assert ( + instance.query("SHOW CREATE USERS B") + == TSV(["CREATE USER B IDENTIFIED WITH no_password"]) + ) assert instance.query("SHOW CREATE USERS A,B") == TSV( - ["CREATE USER A IDENTIFIED WITH no_password", "CREATE USER B IDENTIFIED WITH no_password"] + [ + "CREATE USER A IDENTIFIED WITH no_password", + "CREATE USER B IDENTIFIED WITH no_password" + ] ) assert instance.query("SHOW CREATE USERS") == TSV( [ diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py index bfd93744832..888f9f55b20 100644 --- a/tests/integration/test_settings_profile/test.py +++ b/tests/integration/test_settings_profile/test.py @@ -152,7 +152,10 @@ def test_smoke(): ] instance.query("ALTER USER robin SETTINGS NONE") - assert instance.query("SHOW CREATE USER robin") == "CREATE USER robin IDENTIFIED WITH no_password\n" + assert ( + instance.query("SHOW CREATE USER robin") + == "CREATE USER robin IDENTIFIED WITH no_password\n" + ) assert ( instance.query( "SELECT value FROM system.settings WHERE name = 'max_memory_usage'", diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py index 487a9a2744a..c43c6a2e52a 100644 --- a/tests/integration/test_user_valid_until/test.py +++ b/tests/integration/test_user_valid_until/test.py @@ -22,7 +22,10 @@ def test_basic(started_cluster): # 1. Without VALID UNTIL node.query("CREATE USER user_basic") - assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic IDENTIFIED WITH no_password\n" + assert ( + node.query("SHOW CREATE USER user_basic") + == "CREATE USER user_basic IDENTIFIED WITH no_password\n" + ) assert node.query("SELECT 1", user="user_basic") == "1\n" # 2. With valid VALID UNTIL @@ -48,7 +51,10 @@ def test_basic(started_cluster): # 4. Reset VALID UNTIL node.query("ALTER USER user_basic VALID UNTIL 'infinity'") - assert node.query("SHOW CREATE USER user_basic") == "CREATE USER user_basic IDENTIFIED WITH no_password\n" + assert ( + node.query("SHOW CREATE USER user_basic") + == "CREATE USER user_basic IDENTIFIED WITH no_password\n" + ) assert node.query("SELECT 1", user="user_basic") == "1\n" node.query("DROP USER user_basic") @@ -76,7 +82,9 @@ def test_details(started_cluster): ) # 2. Time only is not supported - node.query("CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL '22:03:40'") + node.query( + "CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL '22:03:40'" + ) until_year = datetime.today().strftime("%Y") From 21de3e29612a1fc929168a048a9bfbe096462868 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 10:05:37 -0300 Subject: [PATCH 036/148] try to fix black --- tests/integration/test_grant_and_revoke/test.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 9fc3dfc249a..8de78163520 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -390,18 +390,16 @@ def test_introspection(): instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) - assert ( - instance.query("SHOW CREATE USERS A") - == TSV(["CREATE USER A IDENTIFIED WITH no_password"]) + assert instance.query("SHOW CREATE USERS A") == TSV( + ["CREATE USER A IDENTIFIED WITH no_password"] ) - assert ( - instance.query("SHOW CREATE USERS B") - == TSV(["CREATE USER B IDENTIFIED WITH no_password"]) + assert instance.query("SHOW CREATE USERS B") == TSV( + ["CREATE USER B IDENTIFIED WITH no_password"] ) assert instance.query("SHOW CREATE USERS A,B") == TSV( [ "CREATE USER A IDENTIFIED WITH no_password", - "CREATE USER B IDENTIFIED WITH no_password" + "CREATE USER B IDENTIFIED WITH no_password", ] ) assert instance.query("SHOW CREATE USERS") == TSV( From b219b9938044da68d773c80439e4a47b504ec3a7 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 11:50:16 -0300 Subject: [PATCH 037/148] small refactor --- src/Access/Authentication.cpp | 201 ++++++++++------------------- src/Access/Authentication.h | 2 +- src/Parsers/tests/gtest_Parser.cpp | 1 - 3 files changed, 67 insertions(+), 137 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index e38bd5abfe5..cb67924475b 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -83,176 +83,107 @@ namespace } #endif - std::vector getAuthenticationMethodsOfType( - const std::vector & authentication_methods, - const std::unordered_set & types) - { - std::vector authentication_methods_of_type; - - for (const auto & authentication_method : authentication_methods) - { - if (types.contains(authentication_method.getType())) - { - authentication_methods_of_type.push_back(authentication_method); - } - } - - return authentication_methods_of_type; - } - bool checkKerberosAuthentication( const GSSAcceptorContext * gss_acceptor_context, - const std::vector & authentication_methods, + const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators) { - auto kerberos_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::KERBEROS}); - for (const auto & kerberos_authentication : kerberos_authentication_methods) - { - if (external_authenticators.checkKerberosCredentials(kerberos_authentication.getKerberosRealm(), *gss_acceptor_context)) - { - return true; - } - } - return false; + return authentication_method.getType() == AuthenticationType::KERBEROS + && external_authenticators.checkKerberosCredentials(authentication_method.getKerberosRealm(), *gss_acceptor_context); } bool checkMySQLAuthentication( const MySQLNative41Credentials * mysql_credentials, - const std::vector & authentication_methods) + const AuthenticationData & authentication_method) { - auto mysql_authentication_methods = getAuthenticationMethodsOfType( - authentication_methods, - {AuthenticationType::PLAINTEXT_PASSWORD, AuthenticationType::DOUBLE_SHA1_PASSWORD}); - - for (const auto & mysql_authentication_method : mysql_authentication_methods) + switch (authentication_method.getType()) { - switch (mysql_authentication_method.getType()) - { - case AuthenticationType::PLAINTEXT_PASSWORD: - if (checkPasswordPlainTextMySQL( - mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), mysql_authentication_method.getPasswordHashBinary())) - { - return true; - } - break; - - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - if (checkPasswordDoubleSHA1MySQL( - mysql_credentials->getScramble(), - mysql_credentials->getScrambledPassword(), - mysql_authentication_method.getPasswordHashBinary())) - { - return true; - } - break; - default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid MySQL authentication type"); - } + case AuthenticationType::PLAINTEXT_PASSWORD: + return checkPasswordPlainTextMySQL( + mysql_credentials->getScramble(), + mysql_credentials->getScrambledPassword(), + authentication_method.getPasswordHashBinary()); + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + return checkPasswordDoubleSHA1MySQL( + mysql_credentials->getScramble(), + mysql_credentials->getScrambledPassword(), + authentication_method.getPasswordHashBinary()); + default: + return false; } - return false; } bool checkBasicAuthentication( const BasicCredentials * basic_credentials, - const std::vector & authentication_methods, + const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) { - auto basic_credentials_authentication_methods = getAuthenticationMethodsOfType( - authentication_methods, - {AuthenticationType::NO_PASSWORD, AuthenticationType::PLAINTEXT_PASSWORD, AuthenticationType::SHA256_PASSWORD, - AuthenticationType::DOUBLE_SHA1_PASSWORD, AuthenticationType::LDAP, AuthenticationType::BCRYPT_PASSWORD, - AuthenticationType::HTTP}); - - for (const auto & auth_method : basic_credentials_authentication_methods) + switch (authentication_method.getType()) { - switch (auth_method.getType()) + case AuthenticationType::NO_PASSWORD: { - case AuthenticationType::NO_PASSWORD: - { - return true; - } - case AuthenticationType::PLAINTEXT_PASSWORD: - if (checkPasswordPlainText(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) - { - return true; - } - break; - - case AuthenticationType::SHA256_PASSWORD: - if (checkPasswordSHA256(basic_credentials->getPassword(), auth_method.getPasswordHashBinary(), auth_method.getSalt())) - { - return true; - } - break; - case AuthenticationType::DOUBLE_SHA1_PASSWORD: - if (checkPasswordDoubleSHA1(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) - { - return true; - } - break; - case AuthenticationType::LDAP: - if (external_authenticators.checkLDAPCredentials(auth_method.getLDAPServerName(), *basic_credentials)) - { - return true; - } - break; - case AuthenticationType::BCRYPT_PASSWORD: - if (checkPasswordBcrypt(basic_credentials->getPassword(), auth_method.getPasswordHashBinary())) - { - return true; - } - break; - case AuthenticationType::HTTP: - if (auth_method.getHTTPAuthenticationScheme() == HTTPAuthenticationScheme::BASIC) - { - return external_authenticators.checkHTTPBasicCredentials( - auth_method.getHTTPAuthenticationServerName(), *basic_credentials, settings); - } - break; - default: - throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid basic authentication type"); + return true; } + case AuthenticationType::PLAINTEXT_PASSWORD: + { + return checkPasswordPlainText(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary()); + } + case AuthenticationType::SHA256_PASSWORD: + { + return checkPasswordSHA256( + basic_credentials->getPassword(), authentication_method.getPasswordHashBinary(), authentication_method.getSalt()); + } + case AuthenticationType::DOUBLE_SHA1_PASSWORD: + { + return checkPasswordDoubleSHA1(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary()); + } + case AuthenticationType::LDAP: + { + return external_authenticators.checkLDAPCredentials(authentication_method.getLDAPServerName(), *basic_credentials); + } + case AuthenticationType::BCRYPT_PASSWORD: + { + return checkPasswordBcrypt(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary()); + } + case AuthenticationType::HTTP: + { + if (authentication_method.getHTTPAuthenticationScheme() == HTTPAuthenticationScheme::BASIC) + { + return external_authenticators.checkHTTPBasicCredentials( + authentication_method.getHTTPAuthenticationServerName(), *basic_credentials, settings); + } + break; + } + default: + break; } + return false; } bool checkSSLCertificateAuthentication( const SSLCertificateCredentials * ssl_certificate_credentials, - const std::vector & authentication_methods) + const AuthenticationData & authentication_method) { - const auto ssl_certificate_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSL_CERTIFICATE}); - for (const auto & auth_method : ssl_certificate_authentication_methods) - { - if (auth_method.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName())) - { - return true; - } - } - return false; + return AuthenticationType::SSL_CERTIFICATE == authentication_method.getType() + && authentication_method.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName()); } #if USE_SSH bool checkSshAuthentication( const SshCredentials * ssh_credentials, - const std::vector & authentication_methods) + const AuthenticationData & authentication_method) { - const auto ssh_authentication_methods = getAuthenticationMethodsOfType(authentication_methods, {AuthenticationType::SSH_KEY}); - for (const auto & auth_method : ssh_authentication_methods) - { - if (checkSshSignature(auth_method.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal())) - { - return true; - } - } - return false; + return AuthenticationType::SSH_KEY == authentication_method.getType() + && checkSshSignature(authentication_method.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal()); } #endif } bool Authentication::areCredentialsValid( const Credentials & credentials, - const std::vector & authentication_methods, + const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings) { @@ -261,28 +192,28 @@ bool Authentication::areCredentialsValid( if (const auto * gss_acceptor_context = typeid_cast(&credentials)) { - return checkKerberosAuthentication(gss_acceptor_context, authentication_methods, external_authenticators); + return checkKerberosAuthentication(gss_acceptor_context, authentication_method, external_authenticators); } if (const auto * mysql_credentials = typeid_cast(&credentials)) { - return checkMySQLAuthentication(mysql_credentials, authentication_methods); + return checkMySQLAuthentication(mysql_credentials, authentication_method); } if (const auto * basic_credentials = typeid_cast(&credentials)) { - return checkBasicAuthentication(basic_credentials, authentication_methods, external_authenticators, settings); + return checkBasicAuthentication(basic_credentials, authentication_method, external_authenticators, settings); } if (const auto * ssl_certificate_credentials = typeid_cast(&credentials)) { - return checkSSLCertificateAuthentication(ssl_certificate_credentials, authentication_methods); + return checkSSLCertificateAuthentication(ssl_certificate_credentials, authentication_method); } #if USE_SSH if (const auto * ssh_credentials = typeid_cast(&credentials)) { - return checkSshAuthentication(ssh_credentials, authentication_methods); + return checkSshAuthentication(ssh_credentials, authentication_method); } #endif diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 13bd487f21a..e895001304d 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -24,7 +24,7 @@ struct Authentication /// returned by the authentication server static bool areCredentialsValid( const Credentials & credentials, - const std::vector & authentication_methods, + const AuthenticationData & authentication_method, const ExternalAuthenticators & external_authenticators, SettingsChanges & settings); diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index b88c8f56029..bd5d39773dd 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -87,7 +87,6 @@ TEST_P(ParserTest, parseQuery) { if (input_text.starts_with("ATTACH")) { - // todo arthur auto salt = (dynamic_cast(ast.get())->auth_data.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } From a948334b8b93a8a7814a4396aa217d735149edac Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 11:53:42 -0300 Subject: [PATCH 038/148] minor --- src/Access/IAccessStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 5b29c07e046..e5512a5b7c8 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -574,7 +574,7 @@ bool IAccessStorage::areCredentialsValid( return false; } - return Authentication::areCredentialsValid(credentials, {authentication_method}, external_authenticators, settings); + return Authentication::areCredentialsValid(credentials, authentication_method, external_authenticators, settings); } bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const From 50da0cb7324ab7bfcf056ab5f2219a00c7cbac23 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 12:14:08 -0300 Subject: [PATCH 039/148] remove definition of logical_error from auth.cpp --- src/Access/Authentication.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index cb67924475b..be11bf7ba27 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -13,10 +13,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} namespace { From 43a91947390ebc90d61500aae841b8d3aa58965f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 14:21:29 -0300 Subject: [PATCH 040/148] remove optional from auth_result --- src/Access/IAccessStorage.cpp | 2 +- src/Access/IAccessStorage.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index e5512a5b7c8..5d5a9b9ce18 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -521,7 +521,7 @@ std::optional IAccessStorage::authenticateImpl( { if (auto user = tryRead(*id)) { - AuthResult auth_result { .user_id = *id, .authentication_data = std::nullopt }; + AuthResult auth_result { .user_id = *id }; if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index df2e8739f7a..353d76536fa 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -35,7 +35,7 @@ struct AuthResult UUID user_id; /// Session settings received from authentication server (if any) SettingsChanges settings{}; - std::optional authentication_data; + AuthenticationData authentication_data {}; }; /// Contains entities, i.e. instances of classes derived from IAccessEntity. From e1119587621f0b4cc898e48ddd61341c22f31cdb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 14:33:28 -0300 Subject: [PATCH 041/148] remmovev optional from auth data in session object --- src/Interpreters/Session.cpp | 2 +- src/Interpreters/Session.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index f56fcb2172f..cdca3f5c049 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -707,7 +707,7 @@ void Session::recordLoginSuccess(ContextPtr login_context) const access->getAccess(), getClientInfo(), user, - *user_authenticated_with); + user_authenticated_with); } notified_session_log_about_login = true; diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 6281df25de3..ab4bc53b6f1 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -113,7 +113,7 @@ private: mutable UserPtr user; std::optional user_id; - std::optional user_authenticated_with; + AuthenticationData user_authenticated_with; ContextMutablePtr session_context; mutable bool query_context_created = false; From eb8a18304f33659766c5590c0887991463d30693 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 27 Jun 2024 17:08:14 -0300 Subject: [PATCH 042/148] rename somem stuff --- src/Client/ClientBase.cpp | 4 +-- .../Access/InterpreterCreateUserQuery.cpp | 30 +++++++++---------- ...InterpreterShowCreateAccessEntityQuery.cpp | 2 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 22 +++++++------- src/Parsers/Access/ASTCreateUserQuery.h | 2 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 4 +-- src/Parsers/tests/gtest_Parser.cpp | 2 +- src/Parsers/tests/gtest_common.cpp | 2 +- src/Storages/System/StorageSystemUsers.cpp | 1 + 9 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 10d77d853f5..3024ee64aea 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1912,9 +1912,9 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (const auto * create_user_query = parsed_query->as()) { - if (!create_user_query->attach && !create_user_query->auth_data.empty()) + if (!create_user_query->attach && !create_user_query->authentication_methods.empty()) { - for (const auto & authentication_method : create_user_query->auth_data) + for (const auto & authentication_method : create_user_query->authentication_methods) { auto password = authentication_method->getPassword(); diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index a1f8eaa4cb0..5d0d2c434bd 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -33,7 +33,7 @@ namespace void updateUserFromQueryImpl( User & user, const ASTCreateUserQuery & query, - const std::vector auth_data, + const std::vector authentication_methods, const std::shared_ptr & override_name, const std::optional & override_default_roles, const std::optional & override_settings, @@ -52,13 +52,13 @@ namespace else if (query.names->size() == 1) user.setName(query.names->front()->toString()); - if (!query.attach && !query.alter && auth_data.empty() && !allow_implicit_no_password) + if (!query.attach && !query.alter && authentication_methods.empty() && !allow_implicit_no_password) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication type NO_PASSWORD must " "be explicitly specified, check the setting allow_implicit_no_password " "in the server configuration"); - if (user.authentication_methods.empty() && auth_data.empty()) + if (user.authentication_methods.empty() && authentication_methods.empty()) { user.authentication_methods.emplace_back(); } @@ -68,7 +68,7 @@ namespace user.authentication_methods.clear(); } - for (const auto & authentication_method : auth_data) + for (const auto & authentication_method : authentication_methods) { user.authentication_methods.emplace_back(authentication_method); } @@ -152,12 +152,12 @@ BlockIO InterpreterCreateUserQuery::execute() bool no_password_allowed = access_control.isNoPasswordAllowed(); bool plaintext_password_allowed = access_control.isPlaintextPasswordAllowed(); - std::vector auth_data; - if (!query.auth_data.empty()) + std::vector authentication_methods; + if (!query.authentication_methods.empty()) { - for (const auto & authentication_method_ast : query.auth_data) + for (const auto & authentication_method_ast : query.authentication_methods) { - auth_data.push_back(AuthenticationData::fromAST(*authentication_method_ast, getContext(), !query.attach)); + authentication_methods.push_back(AuthenticationData::fromAST(*authentication_method_ast, getContext(), !query.attach)); } } @@ -224,7 +224,7 @@ BlockIO InterpreterCreateUserQuery::execute() { auto updated_user = typeid_cast>(entity->clone()); updateUserFromQueryImpl( - *updated_user, query, auth_data, {}, default_roles_from_query, settings_from_query, grantees_from_query, + *updated_user, query, authentication_methods, {}, default_roles_from_query, settings_from_query, grantees_from_query, valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); return updated_user; @@ -245,7 +245,7 @@ BlockIO InterpreterCreateUserQuery::execute() { auto new_user = std::make_shared(); updateUserFromQueryImpl( - *new_user, query, auth_data, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, + *new_user, query, authentication_methods, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); new_users.emplace_back(std::move(new_user)); @@ -286,18 +286,18 @@ BlockIO InterpreterCreateUserQuery::execute() void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password) { - std::vector auth_data; - if (!query.auth_data.empty()) + std::vector authentication_methods; + if (!query.authentication_methods.empty()) { - for (const auto & authentication_method_ast : query.auth_data) + for (const auto & authentication_method_ast : query.authentication_methods) { - auth_data.emplace_back(AuthenticationData::fromAST(*authentication_method_ast, {}, !query.attach)); + authentication_methods.emplace_back(AuthenticationData::fromAST(*authentication_method_ast, {}, !query.attach)); } } updateUserFromQueryImpl(user, query, - auth_data, + authentication_methods, {}, {}, {}, diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index e49cf68727c..ef6ddf1866d 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -66,7 +66,7 @@ namespace for (const auto & authentication_method : user.authentication_methods) { - query->auth_data.push_back(authentication_method.toAST()); + query->authentication_methods.push_back(authentication_method.toAST()); } if (user.valid_until) diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 86b91fcb6f4..c6e78955711 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -18,16 +18,16 @@ namespace << quoteString(new_name); } - void formatAuthenticationData(const std::vector> & auth_data, const IAST::FormatSettings & settings) + void formatAuthenticationData(const std::vector> & authentication_methods, const IAST::FormatSettings & settings) { - auth_data[0]->format(settings); + authentication_methods[0]->format(settings); auto settings_with_additional_authentication_method = settings; settings_with_additional_authentication_method.additional_authentication_method = true; - for (auto i = 1u; i < auth_data.size(); i++) + for (auto i = 1u; i < authentication_methods.size(); i++) { - auth_data[i]->format(settings_with_additional_authentication_method); + authentication_methods[i]->format(settings_with_additional_authentication_method); } } @@ -172,7 +172,7 @@ ASTPtr ASTCreateUserQuery::clone() const { auto res = std::make_shared(*this); res->children.clear(); - res->auth_data.clear(); + res->authentication_methods.clear(); if (names) res->names = std::static_pointer_cast(names->clone()); @@ -189,20 +189,20 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); - if (auth_data.empty()) + if (authentication_methods.empty()) { auto ast = std::make_shared(); ast->type = AuthenticationType::NO_PASSWORD; - res->auth_data.push_back(ast); + res->authentication_methods.push_back(ast); res->children.push_back(ast); } else { - for (const auto & authentication_method : auth_data) + for (const auto & authentication_method : authentication_methods) { auto ast_clone = std::static_pointer_cast(authentication_method->clone()); - res->auth_data.push_back(ast_clone); + res->authentication_methods.push_back(ast_clone); res->children.push_back(ast_clone); } } @@ -243,8 +243,8 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (new_name) formatRenameTo(*new_name, format); - if (!auth_data.empty()) - formatAuthenticationData(auth_data, format); + if (!authentication_methods.empty()) + formatAuthenticationData(authentication_methods, format); if (valid_until) formatValidUntil(*valid_until, format); diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 24af6d00ca4..66a31366993 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -49,7 +49,7 @@ public: std::optional new_name; String storage_name; - std::vector> auth_data; + std::vector> authentication_methods; std::optional hosts; std::optional add_hosts; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 36162086609..13d988e215d 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -609,7 +609,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->cluster = std::move(cluster); query->names = std::move(names); query->new_name = std::move(new_name); - query->auth_data = std::move(auth_data); + query->authentication_methods = std::move(auth_data); query->hosts = std::move(hosts); query->add_hosts = std::move(add_hosts); query->remove_hosts = std::move(remove_hosts); @@ -622,7 +622,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); query->replace_authentication_methods = parsed_identified_with; - for (const auto & authentication_method : query->auth_data) + for (const auto & authentication_method : query->authentication_methods) { query->children.push_back(authentication_method); } diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index bd5d39773dd..e2ccdc96c95 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -87,7 +87,7 @@ TEST_P(ParserTest, parseQuery) { if (input_text.starts_with("ATTACH")) { - auto salt = (dynamic_cast(ast.get())->auth_data.back())->getSalt().value_or(""); + auto salt = (dynamic_cast(ast.get())->authentication_methods.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } else diff --git a/src/Parsers/tests/gtest_common.cpp b/src/Parsers/tests/gtest_common.cpp index bc780e17ffd..0aded8c98f7 100644 --- a/src/Parsers/tests/gtest_common.cpp +++ b/src/Parsers/tests/gtest_common.cpp @@ -64,7 +64,7 @@ TEST_P(ParserKQLTest, parseKQLQuery) if (input_text.starts_with("ATTACH")) { // todo arthur check - auto salt = (dynamic_cast(ast.get())->auth_data.back())->getSalt().value_or(""); + auto salt = (dynamic_cast(ast.get())->authentication_methods.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } else diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index e8557efa9f9..bf2078f7bc1 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -112,6 +112,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto & column_grantees_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_default_database = assert_cast(*res_columns[column_index++]); + // todo arthur check this auto add_row = [&](const String & name, const UUID & id, const String & storage_name, From e9360221b7083d94bc6c0137f97b84b3d20e57cb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 28 Jun 2024 08:40:32 -0300 Subject: [PATCH 043/148] trigger ci From 1cd253e05fcf8c9043b56cf24537bbb1d22d4f4f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 28 Jun 2024 12:48:38 -0300 Subject: [PATCH 044/148] fix integ test --- tests/integration/test_grant_and_revoke/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 8de78163520..498c60a1d40 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -464,8 +464,8 @@ def test_introspection(): assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user="A") expected_access1 = ( - "CREATE USER A\n" - "CREATE USER B\n" + "CREATE USER A IDENTIFIED WITH no_password\n" + "CREATE USER B IDENTIFIED WITH no_password\n" "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE `default`" ) expected_access2 = ( From 27c9bb9b106f0eb7ef31f8c6a49ea206ddf4dc63 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 28 Jun 2024 17:06:06 -0300 Subject: [PATCH 045/148] add missing reference qualifier --- src/Access/IAccessStorage.cpp | 2 +- src/Access/IAccessStorage.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 5d5a9b9ce18..10a0341b301 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -553,7 +553,7 @@ std::optional IAccessStorage::authenticateImpl( } bool IAccessStorage::areCredentialsValid( - const std::string user_name, + const std::string & user_name, time_t valid_until, const AuthenticationData & authentication_method, const Credentials & credentials, diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 353d76536fa..74350bdbd63 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -229,7 +229,7 @@ protected: bool allow_no_password, bool allow_plaintext_password) const; virtual bool areCredentialsValid( - const std::string user_name, + const std::string & user_name, time_t valid_until, const AuthenticationData & authentication_method, const Credentials & credentials, From 306d55f636722ce6098ffca743e8f8e7a8d579f5 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Sat, 29 Jun 2024 18:29:05 -0300 Subject: [PATCH 046/148] add some docs --- .../en/sql-reference/statements/alter/user.md | 19 ++++++++++++++++++- .../sql-reference/statements/create/user.md | 8 +++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index b5c156f56a9..6fff131bba3 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -12,7 +12,7 @@ Syntax: ``` sql ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}] + [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'}] [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] @@ -62,3 +62,20 @@ Allows the user with `john` account to grant his privileges to the user with `ja ``` sql ALTER USER john GRANTEES jack; ``` + +Adds new authentication methods to the user while keeping the existing ones + +``` sql +ALTER USER user ADD IDENTIFIED WITH plaintext_password by '1' ADD IDENTIFIED WITH bcrypt_password by '2' ADD IDENTIFIED WITH plaintext_password by '3' +``` + +Reset authentication methods and adds the ones specified in the query (effect of leading IDENTIFIED without the ADD keyword) + +``` sql +ALTER USER user IDENTIFIED WITH plaintext_password by '1' ADD IDENTIFIED WITH bcrypt_password by '2' ADD IDENTIFIED WITH plaintext_password by '3' +``` + +Reset authentication methods and keep the most recent added one +``` sql +ALTER USER user RESET AUTHENTICATION METHODS TO NEW +``` diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index aee98cfcd10..8e58f6d87ac 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -12,7 +12,7 @@ Syntax: ``` sql CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] + [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] [IN access_storage_type] @@ -144,6 +144,12 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re The available password types are: `plaintext_password`, `sha256_password`, `double_sha1_password`. +7. Multiple authentication methods can be specified: + + ```sql + CREATE USER user1 IDENTIFIED WITH plaintext_password by '1' ADD IDENTIFIED WITH bcrypt_password by '2' ADD IDENTIFIED WITH plaintext_password by '3'' + ``` + ## User Host User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways: From 6f020901a899d30be6f12d1c10fe331393a9b4f3 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Sat, 29 Jun 2024 18:39:01 -0300 Subject: [PATCH 047/148] add a few comments --- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 5d0d2c434bd..b5d5c13d9e4 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -58,11 +58,14 @@ namespace "be explicitly specified, check the setting allow_implicit_no_password " "in the server configuration"); + // if user does not have an authentication method and it has not been specified in the query, + // add a default one if (user.authentication_methods.empty() && authentication_methods.empty()) { user.authentication_methods.emplace_back(); } + // a leading IDENTIFIED WITH will drop existing authentication methods in favor of new ones if (replace_authentication_methods) { user.authentication_methods.clear(); @@ -73,6 +76,7 @@ namespace user.authentication_methods.emplace_back(authentication_method); } + // drop existing ones and keep the most recent if (reset_authentication_methods) { auto backup_authentication_method = user.authentication_methods.back(); From 5a2b0ea1cc7e53a27046ce7c80bb7284ec504909 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 5 Jul 2024 16:08:10 -0300 Subject: [PATCH 048/148] simplify formatting of astauth --- src/Parsers/Access/ASTAuthenticationData.cpp | 4 +--- src/Parsers/Access/ASTCreateUserQuery.cpp | 12 +++++++----- src/Parsers/IAST.h | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index 9c76e73ad90..eb70701d1dc 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -160,9 +160,7 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = AuthenticationTypeInfo::get(*type).name; } - const char * identified_string = settings.additional_authentication_method ? " ADD IDENTIFIED" : " IDENTIFIED"; - - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << identified_string << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); if (!auth_type_name.empty()) { diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 0a4c4af0bb7..0aad4fa8152 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -21,14 +21,16 @@ namespace void formatAuthenticationData(const std::vector> & authentication_methods, const IAST::FormatSettings & settings) { + /* + * The first authentication method must be formatted in the form of: `IDENTIFIED WITH xyz..` + * The remaining ones shall contain the `ADD`: `ADD IDENTIFIED WITH`. + * */ authentication_methods[0]->format(settings); - auto settings_with_additional_authentication_method = settings; - settings_with_additional_authentication_method.additional_authentication_method = true; - - for (auto i = 1u; i < authentication_methods.size(); i++) + for (std::size_t i = 1; i < authentication_methods.size(); i++) { - authentication_methods[i]->format(settings_with_additional_authentication_method); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ADD" << (settings.hilite ? IAST::hilite_none : ""); + authentication_methods[i]->format(settings); } } diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index a74f321eead..ee70fed0f07 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -201,7 +201,6 @@ public: bool show_secrets; /// Show secret parts of the AST (e.g. passwords, encryption keys). char nl_or_ws; /// Newline or whitespace. LiteralEscapingStyle literal_escaping_style; - bool additional_authentication_method; explicit FormatSettings( WriteBuffer & ostr_, @@ -210,8 +209,7 @@ public: bool always_quote_identifiers_ = false, IdentifierQuotingStyle identifier_quoting_style_ = IdentifierQuotingStyle::Backticks, bool show_secrets_ = true, - LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular, - bool additional_authentication_method_ = false) + LiteralEscapingStyle literal_escaping_style_ = LiteralEscapingStyle::Regular) : ostr(ostr_) , one_line(one_line_) , hilite(hilite_) @@ -220,7 +218,6 @@ public: , show_secrets(show_secrets_) , nl_or_ws(one_line ? ' ' : '\n') , literal_escaping_style(literal_escaping_style_) - , additional_authentication_method(additional_authentication_method_) { } @@ -233,7 +230,6 @@ public: , show_secrets(other.show_secrets) , nl_or_ws(other.nl_or_ws) , literal_escaping_style(other.literal_escaping_style) - , additional_authentication_method(other.additional_authentication_method) { } From cd0145113f751228e56500f6d2907433929e9622 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 9 Jul 2024 15:05:24 -0300 Subject: [PATCH 049/148] allow other auth methods to be used if no_password is setup but not allowed --- src/Access/IAccessStorage.cpp | 12 +++--------- src/Access/IAccessStorage.h | 1 - 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 10a0341b301..8e3465f14aa 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -533,7 +533,9 @@ std::optional IAccessStorage::authenticateImpl( auto auth_type = auth_method.getType(); if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) - throwAuthenticationTypeNotAllowed(auth_type); + { + continue; + } if (areCredentialsValid(user->getName(), user->valid_until, auth_method, credentials, external_authenticators, auth_result.settings)) { @@ -756,14 +758,6 @@ void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address throw Exception(ErrorCodes::IP_ADDRESS_NOT_ALLOWED, "Connections from {} are not allowed", address.toString()); } -void IAccessStorage::throwAuthenticationTypeNotAllowed(AuthenticationType auth_type) -{ - throw Exception( - ErrorCodes::AUTHENTICATION_FAILED, - "Authentication type {} is not allowed, check the setting allow_{} in the server configuration", - toString(auth_type), AuthenticationTypeInfo::get(auth_type).name); -} - void IAccessStorage::throwInvalidCredentials() { throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials"); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 74350bdbd63..e193b65e77a 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -252,7 +252,6 @@ protected: [[noreturn]] void throwReadonlyCannotRemove(AccessEntityType type, const String & name) const; [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address); [[noreturn]] static void throwInvalidCredentials(); - [[noreturn]] static void throwAuthenticationTypeNotAllowed(AuthenticationType auth_type); [[noreturn]] void throwBackupNotAllowed() const; [[noreturn]] void throwRestoreNotAllowed() const; From e36776551e95fbbec3c47b6c7560da9afc6c5b64 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 9 Jul 2024 15:58:57 -0300 Subject: [PATCH 050/148] remove unused extern --- src/Access/IAccessStorage.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8e3465f14aa..b3fb87f0c62 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -30,7 +30,6 @@ namespace ErrorCodes extern const int IP_ADDRESS_NOT_ALLOWED; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; - extern const int AUTHENTICATION_FAILED; } From acc2249288ae94510b42e5d0d4cdaad68c5a89c1 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 9 Jul 2024 22:05:04 -0300 Subject: [PATCH 051/148] do not allow no_password to co-exist with other auth methods --- .../Access/InterpreterCreateUserQuery.cpp | 13 +++++++++++-- src/Parsers/Access/ParserCreateUserQuery.cpp | 15 ++++++++++++++- .../03174_add_identified_with.reference | 8 +++++++- .../0_stateless/03174_add_identified_with.sh | 15 +++++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index b5d5c13d9e4..75b0aed24c1 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -65,8 +65,17 @@ namespace user.authentication_methods.emplace_back(); } - // a leading IDENTIFIED WITH will drop existing authentication methods in favor of new ones - if (replace_authentication_methods) + bool has_no_password_authentication_method = std::find_if( + user.authentication_methods.begin(), + user.authentication_methods.end(), + [](const AuthenticationData & authentication_method) + { + return authentication_method.getType() == AuthenticationType::NO_PASSWORD; + }) != user.authentication_methods.end(); + + // 1. a leading IDENTIFIED WITH will drop existing authentication methods in favor of new ones. + // 2. if the user contains an auth method of type NO_PASSWORD and another one is being added, NO_PASSWORD must be dropped + if (replace_authentication_methods || (has_no_password_authentication_method && !authentication_methods.empty())) { user.authentication_methods.clear(); } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 11f79acd979..96294c380a0 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -596,6 +596,19 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec break; } + bool has_no_password_authentication_method = std::find_if( + auth_data.begin(), + auth_data.end(), + [](const std::shared_ptr & ast_authentication_data) + { + return ast_authentication_data->type == AuthenticationType::NO_PASSWORD; + }) != auth_data.end(); + + if (has_no_password_authentication_method && auth_data.size() > 1) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "NO_PASSWORD Authentication method cannot co-exist with other authentication methods"); + } + if (!alter && !hosts) { String common_host_pattern; @@ -630,7 +643,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->valid_until = std::move(valid_until); query->storage_name = std::move(storage_name); query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); - query->replace_authentication_methods = parsed_identified_with; + query->replace_authentication_methods = parsed_identified_with || has_no_password_authentication_method; for (const auto & authentication_method : query->authentication_methods) { diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index e8e1ee9d294..ae330212941 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -37,4 +37,10 @@ CREATE Identified with must precede all add identified with, not allowed BAD_ARGUMENTS Create user with no identification Add identified with -CREATE USER u01_03174 IDENTIFIED WITH no_password ADD IDENTIFIED WITH plaintext_password +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password +Try to provide no_password mixed with other authentication methods, should not be allowed +BAD_ARGUMENTS +Adding no_password, should drop existing auth method +CREATE USER u01_03174 IDENTIFIED WITH no_password +Trying to auth with no pwd, should succeed +1 diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index 0884f31dd73..c3c42047bcf 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -13,6 +13,11 @@ ssh_key="-----BEGIN OPENSSH PRIVATE KEY----- Ux7i7d3xPoseFrwnhY4YAAAADWFydGh1ckBhcnRodXI= -----END OPENSSH PRIVATE KEY-----" +function test_login_no_pwd +{ + ${CLICKHOUSE_CLIENT} --user $1 --query "select 1" +} + function test_login_pwd { ${CLICKHOUSE_CLIENT} --user $1 --password $2 --query "select 1" @@ -107,4 +112,14 @@ ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_p ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" +echo "Try to provide no_password mixed with other authentication methods, should not be allowed" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '8' ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + +echo "Adding no_password, should drop existing auth method" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH no_password" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + +echo "Trying to auth with no pwd, should succeed" +test_login_no_pwd ${user} + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 91e8ef6776402579d79afe8c5eccadf34e2482bb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 10 Jul 2024 09:11:02 -0300 Subject: [PATCH 052/148] do not allow no_pwd to co-exist with other auth methods --- src/Parsers/Access/ParserCreateUserQuery.cpp | 7 +++++++ .../0_stateless/03174_add_identified_with.reference | 5 ++++- tests/queries/0_stateless/03174_add_identified_with.sh | 8 ++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 96294c380a0..3b3e419d615 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -523,6 +523,13 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (parsed_add_new_method) { + if (add_new_auth_method->type == AuthenticationType::NO_PASSWORD) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The authentication method 'no_password' cannot be used with the ADD keyword. " + "Use 'ALTER USER xyz IDENTIFIED WITH no_password' to replace existing authentication methods"); + + } + auth_data.push_back(add_new_auth_method); continue; } diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index ae330212941..cf5e038b58e 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -40,7 +40,10 @@ Add identified with CREATE USER u01_03174 IDENTIFIED WITH plaintext_password Try to provide no_password mixed with other authentication methods, should not be allowed BAD_ARGUMENTS -Adding no_password, should drop existing auth method +Adding no_password, should fail +BAD_ARGUMENTS +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password +Replacing existing authentication methods in favor of no_password, should succeed CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index c3c42047bcf..23e878438ac 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -115,8 +115,12 @@ ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" echo "Try to provide no_password mixed with other authentication methods, should not be allowed" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '8' ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "Adding no_password, should drop existing auth method" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH no_password" +echo "Adding no_password, should fail" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + +echo "Replacing existing authentication methods in favor of no_password, should succeed" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH no_password" ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" echo "Trying to auth with no pwd, should succeed" From 29e0d5c1e333e83da7e15d2c5ddaad57369e44ad Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 11 Jul 2024 17:04:28 -0300 Subject: [PATCH 053/148] use comma separated instead of multiple add id add id --- src/Parsers/Access/ASTAuthenticationData.cpp | 7 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 22 ++- src/Parsers/Access/ParserCreateUserQuery.cpp | 167 +++++++++--------- .../03174_add_identified_with.reference | 12 +- .../0_stateless/03174_add_identified_with.sh | 16 +- 5 files changed, 108 insertions(+), 116 deletions(-) diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp index eb70701d1dc..75082041161 100644 --- a/src/Parsers/Access/ASTAuthenticationData.cpp +++ b/src/Parsers/Access/ASTAuthenticationData.cpp @@ -44,7 +44,7 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt { if (type && *type == AuthenticationType::NO_PASSWORD) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH no_password" + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " no_password" << (settings.hilite ? IAST::hilite_none : ""); return; } @@ -160,12 +160,9 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt auth_type_name = AuthenticationTypeInfo::get(*type).name; } - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); - if (!auth_type_name.empty()) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH " << auth_type_name - << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << auth_type_name << (settings.hilite ? IAST::hilite_none : ""); } if (!prefix.empty()) diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 0aad4fa8152..4eebc471d5e 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -21,16 +21,24 @@ namespace void formatAuthenticationData(const std::vector> & authentication_methods, const IAST::FormatSettings & settings) { - /* - * The first authentication method must be formatted in the form of: `IDENTIFIED WITH xyz..` - * The remaining ones shall contain the `ADD`: `ADD IDENTIFIED WITH`. - * */ - authentication_methods[0]->format(settings); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); - for (std::size_t i = 1; i < authentication_methods.size(); i++) + // safe because this method is only called if authentication_methods.size > 1 + // if the first type is present, include the `WITH` keyword + if (authentication_methods[0]->type) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " WITH" << (settings.hilite ? IAST::hilite_none : ""); + } + + for (std::size_t i = 0; i < authentication_methods.size(); i++) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ADD" << (settings.hilite ? IAST::hilite_none : ""); authentication_methods[i]->format(settings); + + bool is_last = i < authentication_methods.size() - 1; + if (is_last) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : ","); + } } } diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 3b3e419d615..2d5e4eaea98 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -48,21 +48,10 @@ namespace }); } - bool parseAuthenticationData(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data) + bool parseAuthenticationData(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data, bool is_type_specifier_mandatory) { return IParserBase::wrapParseImpl(pos, [&] { - if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) - { - auth_data = std::make_shared(); - auth_data->type = AuthenticationType::NO_PASSWORD; - - return true; - } - - if (!ParserKeyword{Keyword::IDENTIFIED}.ignore(pos, expected)) - return false; - std::optional type; bool expect_password = false; @@ -73,51 +62,48 @@ namespace bool expect_public_ssh_key = false; bool expect_http_auth_server = false; - if (ParserKeyword{Keyword::WITH}.ignore(pos, expected)) + for (auto check_type : collections::range(AuthenticationType::MAX)) { - for (auto check_type : collections::range(AuthenticationType::MAX)) + if (ParserKeyword{AuthenticationTypeInfo::get(check_type).keyword}.ignore(pos, expected)) { - if (ParserKeyword{AuthenticationTypeInfo::get(check_type).keyword}.ignore(pos, expected)) - { - type = check_type; + type = check_type; - if (check_type == AuthenticationType::LDAP) - expect_ldap_server_name = true; - else if (check_type == AuthenticationType::KERBEROS) - expect_kerberos_realm = true; - else if (check_type == AuthenticationType::SSL_CERTIFICATE) - expect_ssl_cert_subjects = true; - else if (check_type == AuthenticationType::SSH_KEY) - expect_public_ssh_key = true; - else if (check_type == AuthenticationType::HTTP) - expect_http_auth_server = true; - else if (check_type != AuthenticationType::NO_PASSWORD) - expect_password = true; + if (check_type == AuthenticationType::LDAP) + expect_ldap_server_name = true; + else if (check_type == AuthenticationType::KERBEROS) + expect_kerberos_realm = true; + else if (check_type == AuthenticationType::SSL_CERTIFICATE) + expect_ssl_cert_subjects = true; + else if (check_type == AuthenticationType::SSH_KEY) + expect_public_ssh_key = true; + else if (check_type == AuthenticationType::HTTP) + expect_http_auth_server = true; + else if (check_type != AuthenticationType::NO_PASSWORD) + expect_password = true; - break; - } + break; } + } - if (!type) + if (!type) + { + if (ParserKeyword{Keyword::SHA256_HASH}.ignore(pos, expected)) { - if (ParserKeyword{Keyword::SHA256_HASH}.ignore(pos, expected)) - { - type = AuthenticationType::SHA256_PASSWORD; - expect_hash = true; - } - else if (ParserKeyword{Keyword::DOUBLE_SHA1_HASH}.ignore(pos, expected)) - { - type = AuthenticationType::DOUBLE_SHA1_PASSWORD; - expect_hash = true; - } - else if (ParserKeyword{Keyword::BCRYPT_HASH}.ignore(pos, expected)) - { - type = AuthenticationType::BCRYPT_PASSWORD; - expect_hash = true; - } - else - return false; + type = AuthenticationType::SHA256_PASSWORD; + expect_hash = true; } + else if (ParserKeyword{Keyword::DOUBLE_SHA1_HASH}.ignore(pos, expected)) + { + type = AuthenticationType::DOUBLE_SHA1_PASSWORD; + expect_hash = true; + } + else if (ParserKeyword{Keyword::BCRYPT_HASH}.ignore(pos, expected)) + { + type = AuthenticationType::BCRYPT_PASSWORD; + expect_hash = true; + } + else if (is_type_specifier_mandatory) + return false; } /// If authentication type is not specified, then the default password type is used @@ -224,6 +210,43 @@ namespace } + bool parseIdentifiedWith(IParserBase::Pos & pos, Expected & expected, std::vector> & authentication_methods) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) + { + authentication_methods.emplace_back(std::make_shared()); + authentication_methods.back()->type = AuthenticationType::NO_PASSWORD; + + return true; + } + + if (!ParserKeyword{Keyword::IDENTIFIED}.ignore(pos, expected)) + return false; + + bool is_type_specifier_mandatory = ParserKeyword{Keyword::WITH}.ignore(pos, expected); + + std::shared_ptr ast_authentication_data; + while (parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory)) + { + authentication_methods.push_back(ast_authentication_data); + + if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) + { + break; + } + + ParserToken{TokenType::Whitespace}.ignore(pos, expected); + + is_type_specifier_mandatory = false; + } + + return !authentication_methods.empty(); + }); + } + + bool parseHostsWithoutPrefix(IParserBase::Pos & pos, Expected & expected, AllowedClientHosts & hosts) { AllowedClientHosts res_hosts; @@ -417,7 +440,7 @@ namespace }); } - bool parseAddNewAuthenticationMethod(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data) + bool parseAddIdentifiedWith(IParserBase::Pos & pos, Expected & expected, std::vector> & auth_data) { return IParserBase::wrapParseImpl(pos, [&] { @@ -426,7 +449,7 @@ namespace return false; } - return parseAuthenticationData(pos, expected, auth_data); + return parseIdentifiedWith(pos, expected, auth_data); }); } @@ -497,41 +520,14 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec while (true) { - std::shared_ptr identified_with_auth_data; - bool parse_auth_data = parseAuthenticationData(pos, expected, identified_with_auth_data); - - bool found_multiple_identified_with = parsed_identified_with && parse_auth_data; - - if (found_multiple_identified_with) + if (auth_data.empty()) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Only one identified with is permitted"); - } + parsed_identified_with = parseIdentifiedWith(pos, expected, auth_data); - if (parse_auth_data) - { - if (parsed_add_new_method) + if (!parsed_identified_with) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Identified with must precede add new auth"); + parsed_add_new_method = parseAddIdentifiedWith(pos, expected, auth_data); } - auth_data.push_back(identified_with_auth_data); - parsed_identified_with = true; - continue; - } - - std::shared_ptr add_new_auth_method; - parsed_add_new_method = parseAddNewAuthenticationMethod(pos, expected, add_new_auth_method); - - if (parsed_add_new_method) - { - if (add_new_auth_method->type == AuthenticationType::NO_PASSWORD) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The authentication method 'no_password' cannot be used with the ADD keyword. " - "Use 'ALTER USER xyz IDENTIFIED WITH no_password' to replace existing authentication methods"); - - } - - auth_data.push_back(add_new_auth_method); - continue; } if (!reset_authentication_methods_to_new.has_value()) @@ -616,6 +612,13 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec throw Exception(ErrorCodes::BAD_ARGUMENTS, "NO_PASSWORD Authentication method cannot co-exist with other authentication methods"); } + if (has_no_password_authentication_method && parsed_add_new_method) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The authentication method 'no_password' cannot be used with the ADD keyword. " + "Use 'ALTER USER xyz IDENTIFIED WITH no_password' to replace existing authentication methods"); + + } + if (!alter && !hosts) { String common_host_pattern; diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_add_identified_with.reference index cf5e038b58e..38f55c1c16d 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_add_identified_with.reference @@ -24,17 +24,9 @@ AUTHENTICATION_FAILED Should work 1 Multiple identified with, not allowed -BAD_ARGUMENTS -Multiple identified with, not allowed, even if mixed -BAD_ARGUMENTS -Identified with must precede all add identified with, not allowed -BAD_ARGUMENTS +Syntax error CREATE Multiple identified with, not allowed -BAD_ARGUMENTS -CREATE Multiple identified with, not allowed, even if mixed -BAD_ARGUMENTS -CREATE Identified with must precede all add identified with, not allowed -BAD_ARGUMENTS +Syntax error Create user with no identification Add identified with CREATE USER u01_03174 IDENTIFIED WITH plaintext_password diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_add_identified_with.sh index 23e878438ac..51414c77684 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_add_identified_with.sh @@ -45,7 +45,7 @@ test_login_pwd_expect_error ${user} '1' echo "New password should work" test_login_pwd ${user} '2' -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password BY '3' ADD IDENTIFIED WITH plaintext_password BY '4'" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password BY '3', plaintext_password BY '4'" echo "Two new passwords were added, should both work" test_login_pwd ${user} '3' @@ -87,20 +87,12 @@ echo "Should work" test_login_pwd ${user} '6' echo "Multiple identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "Multiple identified with, not allowed, even if mixed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password by '7' ADD IDENTIFIED WITH plaintext_password by '8' IDENTIFIED WITH plaintext_password by '9'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "Identified with must precede all add identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" ${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" echo "CREATE Multiple identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "CREATE Multiple identified with, not allowed, even if mixed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '7' ADD IDENTIFIED WITH plaintext_password by '8' IDENTIFIED WITH plaintext_password by '9'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "CREATE Identified with must precede all add identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password by '7' IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" @@ -113,7 +105,7 @@ ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_p ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" echo "Try to provide no_password mixed with other authentication methods, should not be allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '8' ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" echo "Adding no_password, should fail" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" From ee62318348044f82af4bd06aea8d3d2430f9dd0a Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 11 Jul 2024 17:21:33 -0300 Subject: [PATCH 054/148] rename test and add a few more --- ...multiple_authentication_methods.reference} | 8 ++++++++ ... 03174_multiple_authentication_methods.sh} | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) rename tests/queries/0_stateless/{03174_add_identified_with.reference => 03174_multiple_authentication_methods.reference} (68%) rename tests/queries/0_stateless/{03174_add_identified_with.sh => 03174_multiple_authentication_methods.sh} (80%) diff --git a/tests/queries/0_stateless/03174_add_identified_with.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference similarity index 68% rename from tests/queries/0_stateless/03174_add_identified_with.reference rename to tests/queries/0_stateless/03174_multiple_authentication_methods.reference index 38f55c1c16d..27d5df20091 100644 --- a/tests/queries/0_stateless/03174_add_identified_with.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -39,3 +39,11 @@ Replacing existing authentication methods in favor of no_password, should succee CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 +Create user with mix both implicit and explicit auth type, starting with with +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +Create user with mix both implicit and explicit auth type, starting with by +CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password +Use WITH without providing authentication type, should fail +Syntax error +Create user with ADD identification, should fail, add is not allowed for create query +BAD_ARGUMENTS diff --git a/tests/queries/0_stateless/03174_add_identified_with.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh similarity index 80% rename from tests/queries/0_stateless/03174_add_identified_with.sh rename to tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 51414c77684..5503134d880 100755 --- a/tests/queries/0_stateless/03174_add_identified_with.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -119,3 +119,23 @@ echo "Trying to auth with no pwd, should succeed" test_login_no_pwd ${user} ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +echo "Create user with mix both implicit and explicit auth type, starting with with" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4';" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +echo "Create user with mix both implicit and explicit auth type, starting with by" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4';" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +echo "Use WITH without providing authentication type, should fail" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" + +echo "Create user with ADD identification, should fail, add is not allowed for create query" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 300d4ae593ce451efcda316125a4ac570d6c3754 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 11 Jul 2024 17:23:37 -0300 Subject: [PATCH 055/148] add comment back and missing file --- src/Access/Authentication.cpp | 2 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 6d7906a0c85..38ef311acaa 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -120,7 +120,7 @@ namespace { case AuthenticationType::NO_PASSWORD: { - return true; + return true; // N.B. even if the password is not empty! } case AuthenticationType::PLAINTEXT_PASSWORD: { diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 2d5e4eaea98..4da0297dbc7 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -527,6 +527,11 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!parsed_identified_with) { parsed_add_new_method = parseAddIdentifiedWith(pos, expected, auth_data); + + if (parsed_add_new_method && !alter) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Create user query is not allowed to have ADD IDENTIFIED, remove the ADD keyword."); + } } } From 8e0e2cec289296d819811aa0f694566a57f7756e Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 11 Jul 2024 18:06:40 -0300 Subject: [PATCH 056/148] fix hilite test --- .../0_stateless/01316_create_user_syntax_hilite.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference index 48d8b4ee8a1..72e0dd9fb50 100644 --- a/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference +++ b/tests/queries/0_stateless/01316_create_user_syntax_hilite.reference @@ -1 +1 @@ -CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' +CREATE USER user IDENTIFIED WITH plaintext_password BY 'hello' From b428327c6ec3947293b56c545202471446e42e9e Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 12 Jul 2024 09:20:36 -0300 Subject: [PATCH 057/148] trigger ci From 2e9b7e833456fc7312dddd328d351216ab34449f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 12 Jul 2024 10:20:59 -0300 Subject: [PATCH 058/148] Removed stale comment and added log_info about skipped auth methods --- src/Access/IAccessStorage.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index b3fb87f0c62..e8e5d57722d 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -524,15 +524,15 @@ std::optional IAccessStorage::authenticateImpl( if (!isAddressAllowed(*user, address)) throwAddressNotAllowed(address); - // for now, just throw exception in case a user exists with invalid auth method - // back in the day, it would also throw an exception. There might be a smarter alternative - // like a user scan during startup. + bool skipped_not_allowed_authentication_methods = false; + for (const auto & auth_method : user->authentication_methods) { auto auth_type = auth_method.getType(); if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) || ((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password)) { + skipped_not_allowed_authentication_methods = true; continue; } @@ -543,6 +543,12 @@ std::optional IAccessStorage::authenticateImpl( } } + if (skipped_not_allowed_authentication_methods) + { + LOG_INFO(log, "Skipped the check for not allowed authentication methods," + "check allow_no_password and allow_plaintext_password settings in the server configuration"); + } + throwInvalidCredentials(); } } From ca0b821aaf34f8d5cd6ea71c3b491e841601a13f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 12 Jul 2024 14:26:24 -0300 Subject: [PATCH 059/148] fix a few ut --- src/Parsers/tests/gtest_Parser.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index e2ccdc96c95..98197069eb5 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -292,8 +292,8 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "CREATE USER user1 IDENTIFIED WITH no_password" }, { - "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'", - "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'" + "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'", + "CREATE USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'" }, { "CREATE USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", @@ -304,8 +304,8 @@ INSTANTIATE_TEST_SUITE_P(ParserCreateUserQuery, ParserTest, "ALTER USER user1 IDENTIFIED WITH sha256_password BY 'qwe123'" }, { - "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'", - "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123' ADD IDENTIFIED WITH plaintext_password BY 'def123' ADD IDENTIFIED WITH sha256_password BY 'ghi123'" + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'", + "ALTER USER user1 IDENTIFIED WITH plaintext_password BY 'abc123', plaintext_password BY 'def123', sha256_password BY 'ghi123'" }, { "ALTER USER user1 IDENTIFIED WITH sha256_hash BY '7A37B85C8918EAC19A9089C0FA5A2AB4DCE3F90528DCDEEC108B23DDF3607B99' SALT 'salt'", From 3ab676041297e2b7ca50725b985286da9c901979 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 12 Jul 2024 15:21:29 -0300 Subject: [PATCH 060/148] update docs --- docs/en/sql-reference/statements/alter/user.md | 4 ++-- docs/en/sql-reference/statements/create/user.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index 44c1a931261..e58dc9fbd57 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -66,13 +66,13 @@ ALTER USER john GRANTEES jack; Adds new authentication methods to the user while keeping the existing ones ``` sql -ALTER USER user ADD IDENTIFIED WITH plaintext_password by '1' ADD IDENTIFIED WITH bcrypt_password by '2' ADD IDENTIFIED WITH plaintext_password by '3' +ALTER USER user ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' ``` Reset authentication methods and adds the ones specified in the query (effect of leading IDENTIFIED without the ADD keyword) ``` sql -ALTER USER user IDENTIFIED WITH plaintext_password by '1' ADD IDENTIFIED WITH bcrypt_password by '2' ADD IDENTIFIED WITH plaintext_password by '3' +ALTER USER user IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' ``` Reset authentication methods and keep the most recent added one diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 269e67ab1eb..486d5eb3192 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -12,7 +12,7 @@ Syntax: ``` sql CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] + [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] [IN access_storage_type] @@ -147,7 +147,7 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re 7. Multiple authentication methods can be specified: ```sql - CREATE USER user1 IDENTIFIED WITH plaintext_password by '1' ADD IDENTIFIED WITH bcrypt_password by '2' ADD IDENTIFIED WITH plaintext_password by '3'' + CREATE USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'' ``` ## User Host From 006f20858adaba4b9126aa18cfc02a1fd813d967 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 15 Jul 2024 17:01:02 -0300 Subject: [PATCH 061/148] remove dupl inc --- src/Interpreters/Context.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index fc1e87e7b7e..2602afd8b78 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include From 0b151bbe8f5903834368d23e7847a83e0e701055 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 22 Jul 2024 14:31:57 -0300 Subject: [PATCH 062/148] fix conflict --- .../Access/InterpreterCreateUserQuery.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index f9003f266b2..e877d01ead3 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -324,7 +324,20 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat if (query.valid_until) valid_until = getValidUntilFromAST(query.valid_until, {}); - updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, allow_no_password, allow_plaintext_password, true); + updateUserFromQueryImpl( + user, + query, + authentication_methods, + {}, + {}, + {}, + {}, + valid_until, + query.reset_authentication_methods_to_new, + query.replace_authentication_methods, + allow_no_password, + allow_plaintext_password, + true); } void registerInterpreterCreateUserQuery(InterpreterFactory & factory) From 0204de640fae26e64a7b27575b51ff6d723124d0 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 23 Jul 2024 09:32:59 -0300 Subject: [PATCH 063/148] fix test that was recently introduced --- tests/integration/test_user_valid_until/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py index 40d7ceb380e..f25b0021913 100644 --- a/tests/integration/test_user_valid_until/test.py +++ b/tests/integration/test_user_valid_until/test.py @@ -99,14 +99,14 @@ def test_restart(started_cluster): assert ( node.query("SHOW CREATE USER user_restart") - == "CREATE USER user_restart VALID UNTIL \\'2010-11-06 05:03:20\\'\n" + == "CREATE USER user_restart IDENTIFIED WITH no_password VALID UNTIL \\'2010-11-06 05:03:20\\'\n" ) node.restart_clickhouse() assert ( node.query("SHOW CREATE USER user_restart") - == "CREATE USER user_restart VALID UNTIL \\'2010-11-06 05:03:20\\'\n" + == "CREATE USER user_restart IDENTIFIED WITH no_password VALID UNTIL \\'2010-11-06 05:03:20\\'\n" ) error = "Authentication failed" From f3f9d5f4deb0a72cdb9c3fa9f0987bf137427305 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 25 Jul 2024 09:07:13 -0300 Subject: [PATCH 064/148] add notes about no_password behavior --- .../en/sql-reference/statements/alter/user.md | 21 +++++++++++++------ .../sql-reference/statements/create/user.md | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index e58dc9fbd57..959f98b0db1 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -63,19 +63,28 @@ Allows the user with `john` account to grant his privileges to the user with `ja ALTER USER john GRANTEES jack; ``` -Adds new authentication methods to the user while keeping the existing ones +Adds new authentication methods to the user while keeping the existing ones: ``` sql -ALTER USER user ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' +ALTER USER user1 ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' ``` -Reset authentication methods and adds the ones specified in the query (effect of leading IDENTIFIED without the ADD keyword) +Note: `no_password` can not co-exist with other authentication methods for security reasons. +Because of that, it is not possible to `ADD` a `no_password` authentication method. The below query will throw an error: ``` sql -ALTER USER user IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' +ALTER USER user1 ADD IDENTIFIED WITH no_password ``` -Reset authentication methods and keep the most recent added one +If you want to drop authentication methods for a user and rely on `no_password`, you must specify in the below replacing form. + +Reset authentication methods and adds the ones specified in the query (effect of leading IDENTIFIED without the ADD keyword): + ``` sql -ALTER USER user RESET AUTHENTICATION METHODS TO NEW +ALTER USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' +``` + +Reset authentication methods and keep the most recent added one: +``` sql +ALTER USER user1 RESET AUTHENTICATION METHODS TO NEW ``` diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 486d5eb3192..8bfd9f2797b 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -150,6 +150,9 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re CREATE USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'' ``` +Note: `no_password` can not co-exist with other authentication methods for security reasons. Therefore, you can only specify +`no_password` if it is the only authentication method in the query. + ## User Host User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways: From 8eda32600fde5225017aa5ea87ee2ecad551914e Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 25 Jul 2024 09:13:09 -0300 Subject: [PATCH 065/148] remove todo --- src/Parsers/tests/gtest_common.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Parsers/tests/gtest_common.cpp b/src/Parsers/tests/gtest_common.cpp index 0aded8c98f7..594436a1714 100644 --- a/src/Parsers/tests/gtest_common.cpp +++ b/src/Parsers/tests/gtest_common.cpp @@ -63,7 +63,6 @@ TEST_P(ParserKQLTest, parseKQLQuery) { if (input_text.starts_with("ATTACH")) { - // todo arthur check auto salt = (dynamic_cast(ast.get())->authentication_methods.back())->getSalt().value_or(""); EXPECT_TRUE(re2::RE2::FullMatch(salt, expected_ast)); } From 0404a8e80048fd9f0e2d1772774533bc828b44dc Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 25 Jul 2024 14:14:31 -0300 Subject: [PATCH 066/148] make auth_type a vector of int8_t and auth_params a json array --- src/Storages/System/StorageSystemUsers.cpp | 43 +++++++++++----------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index e36c904ded7..105e4af53ef 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -48,8 +49,8 @@ ColumnsDescription StorageSystemUsers::getColumnsDescription() {"name", std::make_shared(), "User name."}, {"id", std::make_shared(), "User ID."}, {"storage", std::make_shared(), "Path to the storage of users. Configured in the access_control_path parameter."}, - {"auth_type", std::make_shared(getAuthenticationTypeEnumValues()), - "Shows the authentication type. " + {"auth_type", std::make_shared(std::make_shared(getAuthenticationTypeEnumValues())), + "Shows the authentication types. " "There are multiple ways of user identification: " "with no password, with plain text password, with SHA256-encoded password, " "with double SHA-1-encoded password or with bcrypt-encoded password." @@ -97,7 +98,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto & column_name = assert_cast(*res_columns[column_index++]); auto & column_id = assert_cast(*res_columns[column_index++]).getData(); auto & column_storage = assert_cast(*res_columns[column_index++]); - auto & column_auth_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_auth_type = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_auth_type_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_auth_params = assert_cast(*res_columns[column_index++]); auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); @@ -119,11 +121,10 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto & column_grantees_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_default_database = assert_cast(*res_columns[column_index++]); - // todo arthur check this auto add_row = [&](const String & name, const UUID & id, const String & storage_name, - const AuthenticationData & auth_data, + const std::vector & authentication_methods, const AllowedClientHosts & allowed_hosts, const RolesOrUsersSet & default_roles, const RolesOrUsersSet & grantees, @@ -132,12 +133,13 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte column_name.insertData(name.data(), name.length()); column_id.push_back(id.toUnderType()); column_storage.insertData(storage_name.data(), storage_name.length()); - column_auth_type.push_back(static_cast(auth_data.getType())); - if (auth_data.getType() == AuthenticationType::LDAP || - auth_data.getType() == AuthenticationType::KERBEROS || - auth_data.getType() == AuthenticationType::SSL_CERTIFICATE) + Poco::JSON::Array json_array; + + for (const auto & auth_data : authentication_methods) { + column_auth_type.insertValue(static_cast(auth_data.getType())); + Poco::JSON::Object auth_params_json; if (auth_data.getType() == AuthenticationType::LDAP) @@ -165,18 +167,17 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auth_params_json.set("subject_alt_names", subject_alt_names); } - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - oss.exceptions(std::ios::failbit); - Poco::JSON::Stringifier::stringify(auth_params_json, oss); - const auto str = oss.str(); + json_array.add(auth_params_json); + } - column_auth_params.insertData(str.data(), str.size()); - } - else - { - static constexpr std::string_view empty_json{"{}"}; - column_auth_params.insertData(empty_json.data(), empty_json.length()); - } + column_auth_type_offsets.push_back(column_auth_type.size()); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json_array, oss); + const auto authentication_params_str = oss.str(); + + column_auth_params.insertData(authentication_params_str.data(), authentication_params_str.size()); if (allowed_hosts.containsAnyHost()) { @@ -248,7 +249,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->authentication_methods.back(), user->allowed_client_hosts, + add_row(user->getName(), id, storage->getStorageName(), user->authentication_methods, user->allowed_client_hosts, user->default_roles, user->grantees, user->default_database); } } From 77d46aad0555b5044a69cb743969ba79effe516b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 25 Jul 2024 14:24:26 -0300 Subject: [PATCH 067/148] change auth_params type from string to array --- src/Storages/System/StorageSystemUsers.cpp | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index 105e4af53ef..ce4950f5e7b 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -55,7 +55,9 @@ ColumnsDescription StorageSystemUsers::getColumnsDescription() "with no password, with plain text password, with SHA256-encoded password, " "with double SHA-1-encoded password or with bcrypt-encoded password." }, - {"auth_params", std::make_shared(), "Authentication parameters in the JSON format depending on the auth_type."}, + {"auth_params", std::make_shared(std::make_shared()), + "Authentication parameters in the JSON format depending on the auth_type." + }, {"host_ip", std::make_shared(std::make_shared()), "IP addresses of hosts that are allowed to connect to the ClickHouse server." }, @@ -100,7 +102,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auto & column_storage = assert_cast(*res_columns[column_index++]); auto & column_auth_type = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_auth_type_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); - auto & column_auth_params = assert_cast(*res_columns[column_index++]); + auto & column_auth_params = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_auth_params_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_names = assert_cast(assert_cast(*res_columns[column_index]).getData()); @@ -134,12 +137,8 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte column_id.push_back(id.toUnderType()); column_storage.insertData(storage_name.data(), storage_name.length()); - Poco::JSON::Array json_array; - for (const auto & auth_data : authentication_methods) { - column_auth_type.insertValue(static_cast(auth_data.getType())); - Poco::JSON::Object auth_params_json; if (auth_data.getType() == AuthenticationType::LDAP) @@ -167,18 +166,18 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, ContextPtr conte auth_params_json.set("subject_alt_names", subject_alt_names); } - json_array.add(auth_params_json); + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(auth_params_json, oss); + const auto authentication_params_str = oss.str(); + + column_auth_params.insertData(authentication_params_str.data(), authentication_params_str.size()); + column_auth_type.insertValue(static_cast(auth_data.getType())); } + column_auth_params_offsets.push_back(column_auth_params.size()); column_auth_type_offsets.push_back(column_auth_type.size()); - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - oss.exceptions(std::ios::failbit); - Poco::JSON::Stringifier::stringify(json_array, oss); - const auto authentication_params_str = oss.str(); - - column_auth_params.insertData(authentication_params_str.data(), authentication_params_str.size()); - if (allowed_hosts.containsAnyHost()) { static constexpr std::string_view str{"::/0"}; From 93cbd4bf9a7cc926ff87281d62d9ae17fd376526 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 25 Jul 2024 17:44:38 -0300 Subject: [PATCH 068/148] update test --- .../0_stateless/02117_show_create_table_system.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 8f62eda9233..c15352ff155 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -1136,8 +1136,8 @@ CREATE TABLE system.users `name` String, `id` UUID, `storage` String, - `auth_type` Enum8('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6, 'bcrypt_password' = 7, 'ssh_key' = 8, 'http' = 9, 'jwt' = 10), - `auth_params` String, + `auth_type` Array(Enum8('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6, 'bcrypt_password' = 7, 'ssh_key' = 8, 'http' = 9, 'jwt' = 10)), + `auth_params` Array(String), `host_ip` Array(String), `host_names` Array(String), `host_names_regexp` Array(String), From 921947e3681879fa6adc1166c4dea9f49073650b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 26 Jul 2024 10:45:25 -0300 Subject: [PATCH 069/148] fix tests and add new one --- tests/integration/test_tlsv1_3/test.py | 4 ++-- tests/queries/0_stateless/01292_create_user.reference | 10 ++++++---- tests/queries/0_stateless/01292_create_user.sql | 5 +++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py index e36989a9cdb..06222d0370a 100644 --- a/tests/integration/test_tlsv1_3/test.py +++ b/tests/integration/test_tlsv1_3/test.py @@ -219,6 +219,6 @@ def test_create_user(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" ) - == 'emma\tssl_certificate\t{"common_names":["client2"]}\n' - 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' + == 'emma\t[\'ssl_certificate\']\t[\'{"common_names":["client2"]}\']\n' + 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index c23723b3ec1..3df69ae2669 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -106,10 +106,10 @@ CREATE USER u2_01292 IDENTIFIED WITH no_password DEFAULT ROLE r1_01292, r2_01292 CREATE USER u3_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 CREATE USER u4_01292 IDENTIFIED WITH no_password HOST LIKE \'%.%.myhost.com\' DEFAULT ROLE r1_01292, r2_01292 -- system.users -u1_01292 local_directory plaintext_password {} [] ['localhost'] [] [] 1 [] [] -u2_01292 local_directory no_password {} [] [] [] ['%.%.myhost.com'] 0 [] [] -u3_01292 local_directory sha256_password {} ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] -u4_01292 local_directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_01292'] +u1_01292 local_directory ['plaintext_password'] ['{}'] [] ['localhost'] [] [] 1 [] [] +u2_01292 local_directory ['no_password'] ['{}'] [] [] [] ['%.%.myhost.com'] 0 [] [] +u3_01292 local_directory ['sha256_password'] ['{}'] ['192.169.1.1','192.168.0.0/16'] ['localhost'] [] [] 0 ['r1_01292'] [] +u4_01292 local_directory ['double_sha1_password'] ['{}'] ['::/0'] [] [] [] 1 [] ['r1_01292'] -- system.settings_profile_elements \N u1_01292 \N 0 readonly 1 \N \N \N \N \N u2_01292 \N 0 \N \N \N \N \N default @@ -117,3 +117,5 @@ u4_01292 local_directory double_sha1_password {} ['::/0'] [] [] [] 1 [] ['r1_012 \N u4_01292 \N 0 \N \N \N \N \N default \N u4_01292 \N 1 max_memory_usage 5000000 \N \N \N \N \N u4_01292 \N 2 readonly 1 \N \N \N \N +-- multiple authentication methods +u1_01292 ['plaintext_password','kerberos','bcrypt_password','ldap'] ['{}','{"realm":"qwerty10"}','{}','{"server":"abc"}'] diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql index 46808aec1ef..41633b3d423 100644 --- a/tests/queries/0_stateless/01292_create_user.sql +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -233,3 +233,8 @@ SELECT * FROM system.settings_profile_elements WHERE user_name LIKE 'u%\_01292' DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292; DROP ROLE r1_01292, r2_01292; + +SELECT '-- multiple authentication methods'; +CREATE USER u1_01292 IDENTIFIED WITH plaintext_password by '1', kerberos REALM 'qwerty10', bcrypt_password by '3', ldap SERVER 'abc'; +SELECT name, auth_type, auth_params FROM system.users WHERE name = 'u1_01292' ORDER BY name; +DROP USER u1_01292; From a21529f66aea514f6b344453cd78415ee22a82d1 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 26 Jul 2024 11:05:05 -0300 Subject: [PATCH 070/148] style --- tests/integration/test_tlsv1_3/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py index 06222d0370a..07239c1b31f 100644 --- a/tests/integration/test_tlsv1_3/test.py +++ b/tests/integration/test_tlsv1_3/test.py @@ -219,6 +219,6 @@ def test_create_user(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" ) - == 'emma\t[\'ssl_certificate\']\t[\'{"common_names":["client2"]}\']\n' + == "emma\t['ssl_certificate']\t['{\"common_names\":[\"client2\"]}']\n" 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) From 5a45563f1bd8580fbd2d9e2a708e047142f689be Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 26 Jul 2024 14:36:58 -0300 Subject: [PATCH 071/148] add note about downgrading --- docs/en/sql-reference/statements/alter/user.md | 4 +++- docs/en/sql-reference/statements/create/user.md | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index 959f98b0db1..9c581655a3c 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -69,7 +69,9 @@ Adds new authentication methods to the user while keeping the existing ones: ALTER USER user1 ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' ``` -Note: `no_password` can not co-exist with other authentication methods for security reasons. +Note: +1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. +2. Note: `no_password` can not co-exist with other authentication methods for security reasons. Because of that, it is not possible to `ADD` a `no_password` authentication method. The below query will throw an error: ``` sql diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 8bfd9f2797b..0329cf0655a 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -150,8 +150,10 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re CREATE USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'' ``` -Note: `no_password` can not co-exist with other authentication methods for security reasons. Therefore, you can only specify -`no_password` if it is the only authentication method in the query. +Note: +1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. +2. `no_password` can not co-exist with other authentication methods for security reasons. Therefore, you can only specify +`no_password` if it is the only authentication method in the query. ## User Host From 456613e7fa1c4a563d6c275d0c75627e480b933d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 26 Jul 2024 14:40:50 -0300 Subject: [PATCH 072/148] minor doc change --- docs/en/sql-reference/statements/alter/user.md | 4 ++-- docs/en/sql-reference/statements/create/user.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index 9c581655a3c..1595f2894df 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -69,9 +69,9 @@ Adds new authentication methods to the user while keeping the existing ones: ALTER USER user1 ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3' ``` -Note: +Notes: 1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. -2. Note: `no_password` can not co-exist with other authentication methods for security reasons. +2. `no_password` can not co-exist with other authentication methods for security reasons. Because of that, it is not possible to `ADD` a `no_password` authentication method. The below query will throw an error: ``` sql diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 0329cf0655a..32b43df9248 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -150,7 +150,7 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re CREATE USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'' ``` -Note: +Notes: 1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. 2. `no_password` can not co-exist with other authentication methods for security reasons. Therefore, you can only specify `no_password` if it is the only authentication method in the query. From 546cca125169fe966da7b210a422c0f0383c5954 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 26 Jul 2024 14:42:58 -0300 Subject: [PATCH 073/148] small comment update --- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index e877d01ead3..b6e68763a3a 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -73,7 +73,7 @@ namespace return authentication_method.getType() == AuthenticationType::NO_PASSWORD; }) != user.authentication_methods.end(); - // 1. a leading IDENTIFIED WITH will drop existing authentication methods in favor of new ones. + // 1. an IDENTIFIED WITH will drop existing authentication methods in favor of new ones. // 2. if the user contains an auth method of type NO_PASSWORD and another one is being added, NO_PASSWORD must be dropped if (replace_authentication_methods || (has_no_password_authentication_method && !authentication_methods.empty())) { From 352b502559528bb0628854e5dbd90a0c2b6a11cb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 26 Jul 2024 14:47:37 -0300 Subject: [PATCH 074/148] update logout session --- src/Interpreters/Session.cpp | 2 +- src/Interpreters/SessionLog.cpp | 10 +++++++--- src/Interpreters/SessionLog.h | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index b65a6bebdf1..a82e8d76c4e 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -304,7 +304,7 @@ Session::~Session() LOG_DEBUG(log, "{} Logout, user_id: {}", toString(auth_id), toString(*user_id)); if (auto session_log = getSessionLog()) { - session_log->addLogOut(auth_id, user, getClientInfo()); + session_log->addLogOut(auth_id, user, user_authenticated_with, getClientInfo()); } } } diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp index 30c3bd570f3..d8c6e3d844f 100644 --- a/src/Interpreters/SessionLog.cpp +++ b/src/Interpreters/SessionLog.cpp @@ -258,15 +258,19 @@ void SessionLog::addLoginFailure( add(std::move(log_entry)); } -void SessionLog::addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info) +void SessionLog::addLogOut( + const UUID & auth_id, + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with, + const ClientInfo & client_info) { auto log_entry = SessionLogElement(auth_id, SESSION_LOGOUT); if (login_user) { log_entry.user = login_user->getName(); - log_entry.user_identified_with = login_user->authentication_methods.back().getType(); + log_entry.user_identified_with = user_authenticated_with.getType(); } - log_entry.external_auth_server = login_user ? login_user->authentication_methods.back().getLDAPServerName() : ""; + log_entry.external_auth_server = user_authenticated_with.getLDAPServerName(); log_entry.client_info = client_info; add(std::move(log_entry)); diff --git a/src/Interpreters/SessionLog.h b/src/Interpreters/SessionLog.h index 2037d6c6861..6221267d14c 100644 --- a/src/Interpreters/SessionLog.h +++ b/src/Interpreters/SessionLog.h @@ -82,7 +82,11 @@ public: const AuthenticationData & user_authenticated_with); void addLoginFailure(const UUID & auth_id, const ClientInfo & info, const std::optional & user, const Exception & reason); - void addLogOut(const UUID & auth_id, const UserPtr & login_user, const ClientInfo & client_info); + void addLogOut( + const UUID & auth_id, + const UserPtr & login_user, + const AuthenticationData & user_authenticated_with, + const ClientInfo & client_info); }; } From f5ee7aaf26046fb943b03de3e6a852119c0b24b8 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Sun, 28 Jul 2024 10:04:38 -0300 Subject: [PATCH 075/148] update some integ tests --- tests/integration/test_grant_and_revoke/test.py | 8 ++++---- tests/integration/test_ssl_cert_authentication/test.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 498c60a1d40..41163e5234a 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -483,8 +483,8 @@ def test_introspection(): [ "A", "local_directory", - "no_password", - "{}", + "['no_password']", + "['{}']", "['::/0']", "[]", "[]", @@ -496,8 +496,8 @@ def test_introspection(): [ "B", "local_directory", - "no_password", - "{}", + "['no_password']", + "['{}']", "['::/0']", "[]", "[]", diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 756a1e1996c..f85f9ec9382 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -335,8 +335,8 @@ def test_create_user(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name IN ['emma', 'lucy'] ORDER BY name" ) - == 'emma\tssl_certificate\t{"common_names":["client2"]}\n' - 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' + == "emma\t['ssl_certificate']\t['{\"common_names\":[\"client2\"]}']\n" + 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) @@ -355,7 +355,7 @@ def test_x509_san_support(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name='jerome'" ) - == 'jerome\tssl_certificate\t{"subject_alt_names":["URI:spiffe:\\\\/\\\\/foo.com\\\\/bar","URI:spiffe:\\\\/\\\\/foo.com\\\\/baz"]}\n' + == 'jerome\t[\'ssl_certificate\']\t[\'{"subject_alt_names":["URI:spiffe:\\\\/\\\\/foo.com\\\\/bar","URI:spiffe:\\\\/\\\\/foo.com\\\\/baz"]}\']\n' ) # user `jerome` is configured via xml config, but `show create` should work regardless. assert ( From c433d9cfdb321f27289df4f35f434a2685af890d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Sun, 28 Jul 2024 12:33:15 -0300 Subject: [PATCH 076/148] retrigger ci From ae72bd57f21d2c049b84b0333f332bce144b46ba Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 29 Jul 2024 09:50:54 -0300 Subject: [PATCH 077/148] try to fix docs? is it broken at all --- docs/en/sql-reference/statements/alter/user.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index 1595f2894df..34a884c4d13 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -70,7 +70,7 @@ ALTER USER user1 ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password ``` Notes: -1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. +1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped. 2. `no_password` can not co-exist with other authentication methods for security reasons. Because of that, it is not possible to `ADD` a `no_password` authentication method. The below query will throw an error: From 4c6aca2eed4da2406c5796bb661cee0a175e5eec Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 12 Aug 2024 15:58:10 -0300 Subject: [PATCH 078/148] add docs about reset auth method --- docs/en/sql-reference/statements/alter/user.md | 1 + docs/en/sql-reference/statements/create/user.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index 34a884c4d13..c5c436b151b 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -15,6 +15,7 @@ ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}] [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] + [RESET AUTHENTICATION METHODS TO NEW] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 32b43df9248..218589391a2 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -15,6 +15,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [VALID UNTIL datetime] + [RESET AUTHENTICATION METHODS TO NEW] [IN access_storage_type] [DEFAULT ROLE role [,...]] [DEFAULT DATABASE database | NONE] From 9abc001296dbd7b8b623cace6d9c651ba9447138 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 13 Aug 2024 10:17:05 -0300 Subject: [PATCH 079/148] fix trailing comma issue --- src/Parsers/Access/ParserCreateUserQuery.cpp | 32 +++++++++++++------ ..._multiple_authentication_methods.reference | 2 ++ .../03174_multiple_authentication_methods.sh | 3 ++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 4da0297dbc7..fdee49f9e80 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -225,21 +225,35 @@ namespace if (!ParserKeyword{Keyword::IDENTIFIED}.ignore(pos, expected)) return false; - bool is_type_specifier_mandatory = ParserKeyword{Keyword::WITH}.ignore(pos, expected); - - std::shared_ptr ast_authentication_data; - while (parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory)) + // Parse first authentication method which doesn't come with a leading comma { - authentication_methods.push_back(ast_authentication_data); + bool is_type_specifier_mandatory = ParserKeyword{Keyword::WITH}.ignore(pos, expected); - if (!ParserToken{TokenType::Comma}.ignore(pos, expected)) + std::shared_ptr ast_authentication_data; + + if (!parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory)) + { + return false; + } + + authentication_methods.push_back(ast_authentication_data); + } + + // Need to save current position, process comma and only update real position in case there is an authentication method after + // the comma. Otherwise, position should not be changed as it needs to be processed by other parsers and possibly throw error + // on trailing comma. + IParserBase::Pos aux_pos = pos; + while (ParserToken{TokenType::Comma}.ignore(aux_pos, expected)) + { + std::shared_ptr ast_authentication_data; + + if (!parseAuthenticationData(aux_pos, expected, ast_authentication_data, false)) { break; } - ParserToken{TokenType::Whitespace}.ignore(pos, expected); - - is_type_specifier_mandatory = false; + pos = aux_pos; + authentication_methods.push_back(ast_authentication_data); } return !authentication_methods.empty(); diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index 27d5df20091..27a84ab9e95 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -47,3 +47,5 @@ Use WITH without providing authentication type, should fail Syntax error Create user with ADD identification, should fail, add is not allowed for create query BAD_ARGUMENTS +Trailing comma should result in syntax error +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 5503134d880..1ea129246cc 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -138,4 +138,7 @@ ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH BY '1';" 2>&1 echo "Create user with ADD identification, should fail, add is not allowed for create query" ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" +echo "Trailing comma should result in syntax error" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 026fa0a7fd73944a0ef930610ba78d1347bc0939 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 14 Aug 2024 12:39:27 -0300 Subject: [PATCH 080/148] fix leading id method without WITH and with type specified being allowed --- src/Parsers/Access/ParserCreateUserQuery.cpp | 15 ++++++++++++--- ...3174_multiple_authentication_methods.reference | 2 ++ .../03174_multiple_authentication_methods.sh | 3 +++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index fdee49f9e80..c36dc48e7a3 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -48,7 +48,12 @@ namespace }); } - bool parseAuthenticationData(IParserBase::Pos & pos, Expected & expected, std::shared_ptr & auth_data, bool is_type_specifier_mandatory) + bool parseAuthenticationData( + IParserBase::Pos & pos, + Expected & expected, + std::shared_ptr & auth_data, + bool is_type_specifier_mandatory, + bool is_type_specifier_allowed) { return IParserBase::wrapParseImpl(pos, [&] { @@ -105,6 +110,10 @@ namespace else if (is_type_specifier_mandatory) return false; } + else if (!is_type_specifier_allowed) + { + return false; + } /// If authentication type is not specified, then the default password type is used if (!type) @@ -231,7 +240,7 @@ namespace std::shared_ptr ast_authentication_data; - if (!parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory)) + if (!parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory, is_type_specifier_mandatory)) { return false; } @@ -247,7 +256,7 @@ namespace { std::shared_ptr ast_authentication_data; - if (!parseAuthenticationData(aux_pos, expected, ast_authentication_data, false)) + if (!parseAuthenticationData(aux_pos, expected, ast_authentication_data, false, true)) { break; } diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index 27a84ab9e95..f30a9566c68 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -49,3 +49,5 @@ Create user with ADD identification, should fail, add is not allowed for create BAD_ARGUMENTS Trailing comma should result in syntax error SYNTAX_ERROR +First auth method can't specify type if WITH keyword is not present +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 1ea129246cc..2945d072583 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -141,4 +141,7 @@ ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_ echo "Trailing comma should result in syntax error" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" +echo "First auth method can't specify type if WITH keyword is not present" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 3247f3ad08f1908df5c58f81b59b1b805c152fae Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 14 Aug 2024 12:44:43 -0300 Subject: [PATCH 081/148] make sure reset authentication methods can only be used on alter queries --- src/Parsers/Access/ParserCreateUserQuery.cpp | 5 +++++ .../03174_multiple_authentication_methods.reference | 2 ++ .../0_stateless/03174_multiple_authentication_methods.sh | 3 +++ 3 files changed, 10 insertions(+) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index c36dc48e7a3..466fd544c63 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -561,6 +561,11 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!reset_authentication_methods_to_new.has_value()) { reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); + + if (reset_authentication_methods_to_new.value() && !alter) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "RESET AUTHENTICATION METHODS TO NEW can only be used on ALTER statement"); + } } if (!valid_until) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index f30a9566c68..8cdc1f9d613 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -51,3 +51,5 @@ Trailing comma should result in syntax error SYNTAX_ERROR First auth method can't specify type if WITH keyword is not present SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement +BAD_ARGUMENTS diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 2945d072583..5e4c5048d55 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -144,4 +144,7 @@ ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_p echo "First auth method can't specify type if WITH keyword is not present" ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" +echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 17c1cef52b9ea08e85fab108d590c5e1cf3e2e02 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 15 Aug 2024 10:30:33 -0300 Subject: [PATCH 082/148] add server setting --- src/Access/AccessEntityIO.cpp | 2 +- src/Core/ServerSettings.h | 1 + src/Core/Settings.h | 1 - .../Access/InterpreterCreateUserQuery.cpp | 28 +++++++++++++++---- .../Access/InterpreterCreateUserQuery.h | 7 ++++- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/Access/AccessEntityIO.cpp b/src/Access/AccessEntityIO.cpp index 1b073329296..53b073cb750 100644 --- a/src/Access/AccessEntityIO.cpp +++ b/src/Access/AccessEntityIO.cpp @@ -82,7 +82,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition) if (res) throw Exception(ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION, "Two access entities attached in the same file"); res = user = std::make_unique(); - InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true); + InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true, /* max_number_of_authentication_methods = */ std::numeric_limits::max()); } else if (auto * create_role_query = query->as()) { diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index d13e6251ca9..887beba4b13 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -116,6 +116,7 @@ namespace DB M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \ M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \ + M(UInt64, max_authentication_methods_per_user, 256, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users.", 0) \ M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \ M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index acdc8316a4d..d27c9fce7bb 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -966,7 +966,6 @@ class IColumn; \ M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \ M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \ - \ /** Experimental feature for moving data between shards. */ \ M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \ diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index b6e68763a3a..5178ad184d6 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -43,7 +44,8 @@ namespace bool replace_authentication_methods, bool allow_implicit_no_password, bool allow_no_password, - bool allow_plaintext_password) + bool allow_plaintext_password, + std::size_t max_number_of_authentication_methods) { if (override_name) user.setName(override_name->toString()); @@ -80,6 +82,14 @@ namespace user.authentication_methods.clear(); } + auto number_of_authentication_methods = user.authentication_methods.size() + authentication_methods.size(); + if (number_of_authentication_methods > max_number_of_authentication_methods) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user." + "Check the `max_authentication_methods_per_user` setting"); + } + for (const auto & authentication_method : authentication_methods) { user.authentication_methods.emplace_back(authentication_method); @@ -251,7 +261,8 @@ BlockIO InterpreterCreateUserQuery::execute() updateUserFromQueryImpl( *updated_user, query, authentication_methods, {}, default_roles_from_query, settings_from_query, grantees_from_query, valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, - implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + implicit_no_password_allowed, no_password_allowed, + plaintext_password_allowed, getContext()->getServerSettings().max_authentication_methods_per_user); return updated_user; }; @@ -272,7 +283,8 @@ BlockIO InterpreterCreateUserQuery::execute() updateUserFromQueryImpl( *new_user, query, authentication_methods, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}, valid_until, query.reset_authentication_methods_to_new, query.replace_authentication_methods, - implicit_no_password_allowed, no_password_allowed, plaintext_password_allowed); + implicit_no_password_allowed, no_password_allowed, + plaintext_password_allowed, getContext()->getServerSettings().max_authentication_methods_per_user); new_users.emplace_back(std::move(new_user)); } @@ -309,7 +321,12 @@ BlockIO InterpreterCreateUserQuery::execute() } -void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password) +void InterpreterCreateUserQuery::updateUserFromQuery( + User & user, + const ASTCreateUserQuery & query, + bool allow_no_password, + bool allow_plaintext_password, + std::size_t max_number_of_authentication_methods) { std::vector authentication_methods; if (!query.authentication_methods.empty()) @@ -337,7 +354,8 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat query.replace_authentication_methods, allow_no_password, allow_plaintext_password, - true); + true, + max_number_of_authentication_methods); } void registerInterpreterCreateUserQuery(InterpreterFactory & factory) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.h b/src/Interpreters/Access/InterpreterCreateUserQuery.h index 372066cfd5e..fea87d33703 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.h +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.h @@ -17,7 +17,12 @@ public: BlockIO execute() override; - static void updateUserFromQuery(User & user, const ASTCreateUserQuery & query, bool allow_no_password, bool allow_plaintext_password); + static void updateUserFromQuery( + User & user, + const ASTCreateUserQuery & query, + bool allow_no_password, + bool allow_plaintext_password, + std::size_t max_number_of_authentication_methods); private: ASTPtr query_ptr; From 72f1695014919c62c048afc6d1016655af3cdf09 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 15 Aug 2024 12:03:30 -0300 Subject: [PATCH 083/148] add integ tests for new setting --- .../__init__.py | 0 .../configs/max_auth_limited.xml | 3 ++ .../test.py | 49 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 tests/integration/test_max_authentication_methods_per_user/__init__.py create mode 100644 tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml create mode 100644 tests/integration/test_max_authentication_methods_per_user/test.py diff --git a/tests/integration/test_max_authentication_methods_per_user/__init__.py b/tests/integration/test_max_authentication_methods_per_user/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml b/tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml new file mode 100644 index 00000000000..c6d4b0077be --- /dev/null +++ b/tests/integration/test_max_authentication_methods_per_user/configs/max_auth_limited.xml @@ -0,0 +1,3 @@ + + 2 + \ No newline at end of file diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py new file mode 100644 index 00000000000..81d36dfd21d --- /dev/null +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -0,0 +1,49 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +limited_node = cluster.add_instance( + "limited_node", + main_configs=["configs/max_auth_limited.xml"], +) + +default_node = cluster.add_instance( + "default_node", +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_create(started_cluster): + expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + assert expected_error in limited_node.query_and_get_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2', BY '3'") + + + assert expected_error not in limited_node.query_and_get_answer_with_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2'") + + limited_node.query("DROP USER u_max_authentication_methods") + +def test_alter(started_cluster): + limited_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") + + expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + assert expected_error in limited_node.query_and_get_error("ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2', BY '3'") + + expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + assert expected_error in limited_node.query_and_get_error("ALTER USER u_max_authentication_methods IDENTIFIED BY '3', BY '4', BY '5'") + + assert expected_error not in limited_node.query_and_get_answer_with_error("ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2'") + + assert expected_error not in limited_node.query_and_get_answer_with_error("ALTER USER u_max_authentication_methods IDENTIFIED BY '2', BY '3'") + + limited_node.query("DROP USER u_max_authentication_methods") From 714a4d871c6abbaab19b48076e6e395cfd5fcf8f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 15 Aug 2024 12:18:09 -0300 Subject: [PATCH 084/148] add integ tests for defa7ult value --- .../test.py | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py index 81d36dfd21d..2a1d54d9b5a 100644 --- a/tests/integration/test_max_authentication_methods_per_user/test.py +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -23,11 +23,11 @@ def started_cluster(): finally: cluster.shutdown() +expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" def test_create(started_cluster): - expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" - assert expected_error in limited_node.query_and_get_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2', BY '3'") + assert expected_error in limited_node.query_and_get_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2', BY '3'") assert expected_error not in limited_node.query_and_get_answer_with_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2'") @@ -36,10 +36,8 @@ def test_create(started_cluster): def test_alter(started_cluster): limited_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") - expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" assert expected_error in limited_node.query_and_get_error("ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2', BY '3'") - expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" assert expected_error in limited_node.query_and_get_error("ALTER USER u_max_authentication_methods IDENTIFIED BY '3', BY '4', BY '5'") assert expected_error not in limited_node.query_and_get_answer_with_error("ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2'") @@ -47,3 +45,44 @@ def test_alter(started_cluster): assert expected_error not in limited_node.query_and_get_answer_with_error("ALTER USER u_max_authentication_methods IDENTIFIED BY '2', BY '3'") limited_node.query("DROP USER u_max_authentication_methods") + + +def get_query_with_multiple_identified_with(operation, username, identified_with_count, add_operation = ""): + identified_clauses = ", ".join([f"BY '1'" for _ in range(identified_with_count)]) + query = f"{operation} USER {username} {add_operation} IDENTIFIED {identified_clauses}" + return query + + +def test_create_default_setting(started_cluster): + expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + + query_exceeds = get_query_with_multiple_identified_with("CREATE", "u_max_authentication_methods", 257) + + assert expected_error in default_node.query_and_get_error(query_exceeds) + + query_not_exceeds = get_query_with_multiple_identified_with("CREATE", "u_max_authentication_methods", 256) + + assert expected_error not in default_node.query_and_get_answer_with_error(query_not_exceeds) + + default_node.query("DROP USER u_max_authentication_methods") + +def test_alter_default_setting(started_cluster): + default_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") + + query_add_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 256, "ADD") + + assert expected_error in default_node.query_and_get_error(query_add_exceeds) + + query_replace_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 257) + + assert expected_error in default_node.query_and_get_error(query_replace_exceeds) + + query_add_not_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 1, "ADD") + + assert expected_error not in default_node.query_and_get_answer_with_error(query_add_not_exceeds) + + query_replace_not_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 2) + + assert expected_error not in default_node.query_and_get_answer_with_error(query_replace_not_exceeds) + + default_node.query("DROP USER u_max_authentication_methods") From 12e6645058a78f3dd74bdbd6982ad9ee49586e7b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 15 Aug 2024 12:19:36 -0300 Subject: [PATCH 085/148] fix black --- .../test.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py index 2a1d54d9b5a..4c20f9cf376 100644 --- a/tests/integration/test_max_authentication_methods_per_user/test.py +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -27,22 +27,34 @@ expected_error = "User can not be created/updated because it exceeds the allowed def test_create(started_cluster): - assert expected_error in limited_node.query_and_get_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2', BY '3'") + assert expected_error in limited_node.query_and_get_error( + "CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2', BY '3'" + ) - assert expected_error not in limited_node.query_and_get_answer_with_error("CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2'") + assert expected_error not in limited_node.query_and_get_answer_with_error( + "CREATE USER u_max_authentication_methods IDENTIFIED BY '1', BY '2'" + ) limited_node.query("DROP USER u_max_authentication_methods") def test_alter(started_cluster): limited_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") - assert expected_error in limited_node.query_and_get_error("ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2', BY '3'") + assert expected_error in limited_node.query_and_get_error( + "ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2', BY '3'" + ) - assert expected_error in limited_node.query_and_get_error("ALTER USER u_max_authentication_methods IDENTIFIED BY '3', BY '4', BY '5'") + assert expected_error in limited_node.query_and_get_error( + "ALTER USER u_max_authentication_methods IDENTIFIED BY '3', BY '4', BY '5'" + ) - assert expected_error not in limited_node.query_and_get_answer_with_error("ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2'") + assert expected_error not in limited_node.query_and_get_answer_with_error( + "ALTER USER u_max_authentication_methods ADD IDENTIFIED BY '2'" + ) - assert expected_error not in limited_node.query_and_get_answer_with_error("ALTER USER u_max_authentication_methods IDENTIFIED BY '2', BY '3'") + assert expected_error not in limited_node.query_and_get_answer_with_error( + "ALTER USER u_max_authentication_methods IDENTIFIED BY '2', BY '3'" + ) limited_node.query("DROP USER u_max_authentication_methods") From 70e7c4e63d58c67d77ebb7e979cc41e35a8025b1 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 15 Aug 2024 12:21:06 -0300 Subject: [PATCH 086/148] fix black with script --- .../test.py | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py index 4c20f9cf376..fb02ef73bf6 100644 --- a/tests/integration/test_max_authentication_methods_per_user/test.py +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -23,8 +23,10 @@ def started_cluster(): finally: cluster.shutdown() + expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" + def test_create(started_cluster): assert expected_error in limited_node.query_and_get_error( @@ -37,6 +39,7 @@ def test_create(started_cluster): limited_node.query("DROP USER u_max_authentication_methods") + def test_alter(started_cluster): limited_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") @@ -59,42 +62,65 @@ def test_alter(started_cluster): limited_node.query("DROP USER u_max_authentication_methods") -def get_query_with_multiple_identified_with(operation, username, identified_with_count, add_operation = ""): +def get_query_with_multiple_identified_with( + operation, username, identified_with_count, add_operation="" +): identified_clauses = ", ".join([f"BY '1'" for _ in range(identified_with_count)]) - query = f"{operation} USER {username} {add_operation} IDENTIFIED {identified_clauses}" + query = ( + f"{operation} USER {username} {add_operation} IDENTIFIED {identified_clauses}" + ) return query def test_create_default_setting(started_cluster): expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" - query_exceeds = get_query_with_multiple_identified_with("CREATE", "u_max_authentication_methods", 257) + query_exceeds = get_query_with_multiple_identified_with( + "CREATE", "u_max_authentication_methods", 257 + ) assert expected_error in default_node.query_and_get_error(query_exceeds) - query_not_exceeds = get_query_with_multiple_identified_with("CREATE", "u_max_authentication_methods", 256) + query_not_exceeds = get_query_with_multiple_identified_with( + "CREATE", "u_max_authentication_methods", 256 + ) - assert expected_error not in default_node.query_and_get_answer_with_error(query_not_exceeds) + assert expected_error not in default_node.query_and_get_answer_with_error( + query_not_exceeds + ) default_node.query("DROP USER u_max_authentication_methods") + def test_alter_default_setting(started_cluster): default_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") - query_add_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 256, "ADD") + query_add_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 256, "ADD" + ) assert expected_error in default_node.query_and_get_error(query_add_exceeds) - query_replace_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 257) + query_replace_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 257 + ) assert expected_error in default_node.query_and_get_error(query_replace_exceeds) - query_add_not_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 1, "ADD") + query_add_not_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 1, "ADD" + ) - assert expected_error not in default_node.query_and_get_answer_with_error(query_add_not_exceeds) + assert expected_error not in default_node.query_and_get_answer_with_error( + query_add_not_exceeds + ) - query_replace_not_exceeds = get_query_with_multiple_identified_with("ALTER", "u_max_authentication_methods", 2) + query_replace_not_exceeds = get_query_with_multiple_identified_with( + "ALTER", "u_max_authentication_methods", 2 + ) - assert expected_error not in default_node.query_and_get_answer_with_error(query_replace_not_exceeds) + assert expected_error not in default_node.query_and_get_answer_with_error( + query_replace_not_exceeds + ) default_node.query("DROP USER u_max_authentication_methods") From 770804ffdca82552516ac9e7892c955d61dee1f3 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 16 Aug 2024 08:57:11 -0300 Subject: [PATCH 087/148] change default setting value to 100 --- src/Core/ServerSettings.h | 2 +- .../test_max_authentication_methods_per_user/test.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 887beba4b13..29a45d23163 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -116,7 +116,7 @@ namespace DB M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \ M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \ - M(UInt64, max_authentication_methods_per_user, 256, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users.", 0) \ + M(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users.", 0) \ M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \ M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ \ diff --git a/tests/integration/test_max_authentication_methods_per_user/test.py b/tests/integration/test_max_authentication_methods_per_user/test.py index fb02ef73bf6..0142c7db746 100644 --- a/tests/integration/test_max_authentication_methods_per_user/test.py +++ b/tests/integration/test_max_authentication_methods_per_user/test.py @@ -76,13 +76,13 @@ def test_create_default_setting(started_cluster): expected_error = "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user" query_exceeds = get_query_with_multiple_identified_with( - "CREATE", "u_max_authentication_methods", 257 + "CREATE", "u_max_authentication_methods", 101 ) assert expected_error in default_node.query_and_get_error(query_exceeds) query_not_exceeds = get_query_with_multiple_identified_with( - "CREATE", "u_max_authentication_methods", 256 + "CREATE", "u_max_authentication_methods", 100 ) assert expected_error not in default_node.query_and_get_answer_with_error( @@ -96,19 +96,19 @@ def test_alter_default_setting(started_cluster): default_node.query("CREATE USER u_max_authentication_methods IDENTIFIED BY '1'") query_add_exceeds = get_query_with_multiple_identified_with( - "ALTER", "u_max_authentication_methods", 256, "ADD" + "ALTER", "u_max_authentication_methods", 100, "ADD" ) assert expected_error in default_node.query_and_get_error(query_add_exceeds) query_replace_exceeds = get_query_with_multiple_identified_with( - "ALTER", "u_max_authentication_methods", 257 + "ALTER", "u_max_authentication_methods", 101 ) assert expected_error in default_node.query_and_get_error(query_replace_exceeds) query_add_not_exceeds = get_query_with_multiple_identified_with( - "ALTER", "u_max_authentication_methods", 1, "ADD" + "ALTER", "u_max_authentication_methods", 99, "ADD" ) assert expected_error not in default_node.query_and_get_answer_with_error( @@ -116,7 +116,7 @@ def test_alter_default_setting(started_cluster): ) query_replace_not_exceeds = get_query_with_multiple_identified_with( - "ALTER", "u_max_authentication_methods", 2 + "ALTER", "u_max_authentication_methods", 100 ) assert expected_error not in default_node.query_and_get_answer_with_error( From 01f5337f69f7b78a8962b640731ab6d2869b56ac Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 19 Aug 2024 15:13:39 -0300 Subject: [PATCH 088/148] fix serialization to fix on cluster commands --- src/Parsers/Access/ASTCreateUserQuery.cpp | 16 +++++++++++++--- src/Parsers/Access/ASTCreateUserQuery.h | 1 + src/Parsers/Access/ParserCreateUserQuery.cpp | 17 +++++++++-------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 4eebc471d5e..8f44cb3c409 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -21,8 +21,6 @@ namespace void formatAuthenticationData(const std::vector> & authentication_methods, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (settings.hilite ? IAST::hilite_none : ""); - // safe because this method is only called if authentication_methods.size > 1 // if the first type is present, include the `WITH` keyword if (authentication_methods[0]->type) @@ -200,7 +198,8 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); - if (authentication_methods.empty()) + // If identification (auth method) is missing from query, we should serialize it in the form of `NO_PASSWORD` unless it is alter query + if (!alter && authentication_methods.empty()) { auto ast = std::make_shared(); ast->type = AuthenticationType::NO_PASSWORD; @@ -255,7 +254,15 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & formatRenameTo(*new_name, format); if (!authentication_methods.empty()) + { + if (add_identified_with) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " ADD" << (format.hilite ? IAST::hilite_none : ""); + } + + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED" << (format.hilite ? IAST::hilite_none : ""); formatAuthenticationData(authentication_methods, format); + } if (valid_until) formatValidUntil(*valid_until, format); @@ -278,6 +285,9 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (grantees) formatGrantees(*grantees, format); + + if (reset_authentication_methods_to_new) + format.ostr << (format.hilite ? hilite_keyword : "") << " RESET AUTHENTICATION METHODS TO NEW" << (format.hilite ? hilite_none : ""); } } diff --git a/src/Parsers/Access/ASTCreateUserQuery.h b/src/Parsers/Access/ASTCreateUserQuery.h index 66a31366993..e1bae98f2f3 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.h +++ b/src/Parsers/Access/ASTCreateUserQuery.h @@ -43,6 +43,7 @@ public: bool if_not_exists = false; bool or_replace = false; bool reset_authentication_methods_to_new = false; + bool add_identified_with = false; bool replace_authentication_methods = false; std::shared_ptr names; diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 466fd544c63..0ef658d2377 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -536,10 +536,10 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec ASTPtr valid_until; String cluster; String storage_name; - std::optional reset_authentication_methods_to_new; + bool reset_authentication_methods_to_new = false; bool parsed_identified_with = false; - bool parsed_add_new_method = false; + bool parsed_add_identified_with = false; while (true) { @@ -549,20 +549,20 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (!parsed_identified_with) { - parsed_add_new_method = parseAddIdentifiedWith(pos, expected, auth_data); + parsed_add_identified_with = parseAddIdentifiedWith(pos, expected, auth_data); - if (parsed_add_new_method && !alter) + if (parsed_add_identified_with && !alter) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Create user query is not allowed to have ADD IDENTIFIED, remove the ADD keyword."); } } } - if (!reset_authentication_methods_to_new.has_value()) + if (!reset_authentication_methods_to_new) { reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); - if (reset_authentication_methods_to_new.value() && !alter) + if (reset_authentication_methods_to_new && !alter) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "RESET AUTHENTICATION METHODS TO NEW can only be used on ALTER statement"); } @@ -645,7 +645,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec throw Exception(ErrorCodes::BAD_ARGUMENTS, "NO_PASSWORD Authentication method cannot co-exist with other authentication methods"); } - if (has_no_password_authentication_method && parsed_add_new_method) + if (has_no_password_authentication_method && parsed_add_identified_with) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "The authentication method 'no_password' cannot be used with the ADD keyword. " "Use 'ALTER USER xyz IDENTIFIED WITH no_password' to replace existing authentication methods"); @@ -685,7 +685,8 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->default_database = std::move(default_database); query->valid_until = std::move(valid_until); query->storage_name = std::move(storage_name); - query->reset_authentication_methods_to_new = reset_authentication_methods_to_new.value_or(false); + query->reset_authentication_methods_to_new = reset_authentication_methods_to_new; + query->add_identified_with = parsed_add_identified_with; query->replace_authentication_methods = parsed_identified_with || has_no_password_authentication_method; for (const auto & authentication_method : query->authentication_methods) From 623c507e5febdee3f220797799c20d921fd8a9a9 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 19 Aug 2024 16:42:48 -0300 Subject: [PATCH 089/148] fix in the right place.. --- src/Parsers/Access/ASTCreateUserQuery.cpp | 30 ++++++++++------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/Parsers/Access/ASTCreateUserQuery.cpp b/src/Parsers/Access/ASTCreateUserQuery.cpp index 8f44cb3c409..ec48c32b684 100644 --- a/src/Parsers/Access/ASTCreateUserQuery.cpp +++ b/src/Parsers/Access/ASTCreateUserQuery.cpp @@ -198,23 +198,11 @@ ASTPtr ASTCreateUserQuery::clone() const if (settings) res->settings = std::static_pointer_cast(settings->clone()); - // If identification (auth method) is missing from query, we should serialize it in the form of `NO_PASSWORD` unless it is alter query - if (!alter && authentication_methods.empty()) + for (const auto & authentication_method : authentication_methods) { - auto ast = std::make_shared(); - ast->type = AuthenticationType::NO_PASSWORD; - - res->authentication_methods.push_back(ast); - res->children.push_back(ast); - } - else - { - for (const auto & authentication_method : authentication_methods) - { - auto ast_clone = std::static_pointer_cast(authentication_method->clone()); - res->authentication_methods.push_back(ast_clone); - res->children.push_back(ast_clone); - } + auto ast_clone = std::static_pointer_cast(authentication_method->clone()); + res->authentication_methods.push_back(ast_clone); + res->children.push_back(ast_clone); } return res; @@ -253,7 +241,15 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (new_name) formatRenameTo(*new_name, format); - if (!authentication_methods.empty()) + if (authentication_methods.empty()) + { + // If identification (auth method) is missing from query, we should serialize it in the form of `NO_PASSWORD` unless it is alter query + if (!alter) + { + format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH no_password" << (format.hilite ? IAST::hilite_none : ""); + } + } + else { if (add_identified_with) { From 2fcbe2465a44708fc500a3f6123dd575f2f07e5c Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 20 Aug 2024 09:01:43 +0000 Subject: [PATCH 090/148] Fix style in Functions/printf.cpp --- src/Functions/printf.cpp | 18 ++++++++---------- ..._new_functions_must_be_documented.reference | 1 - 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp index 3cf3efaf534..afacf164a00 100644 --- a/src/Functions/printf.cpp +++ b/src/Functions/printf.cpp @@ -50,13 +50,6 @@ private: return executeNonconstant(input); } - [[maybe_unused]] String toString() const - { - WriteBufferFromOwnString buf; - buf << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure() << "\n"; - return buf.str(); - } - private: ColumnWithTypeAndName executeLiteral(std::string_view literal) const { @@ -231,9 +224,7 @@ public: const auto & instruction = instructions[i]; try { - // std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl; concat_args[i] = instruction.execute(); - // std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl; } catch (const fmt::v9::format_error & e) { @@ -358,7 +349,14 @@ private: REGISTER_FUNCTION(Printf) { - factory.registerFunction(); + factory.registerFunction( + FunctionDocumentation{.description=R"( +The `printf` function formats the given string with the values (strings, integers, floating-points etc.) listed in the arguments, similar to printf function in C++. +The format string can contain format specifiers starting with `%` character. +Anything not contained in `%` and the following format specifier is considered literal text and copied verbatim into the output. +Literal `%` character can be escaped by `%%`.)", .examples{{"sum", "select printf('%%%s %s %d', 'Hello', 'World', 2024);", "%Hello World 2024"}}, .categories{"String"} +}); + } } diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference index c39f1fb1ce9..8dd8910c858 100644 --- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference +++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference @@ -562,7 +562,6 @@ positionCaseInsensitive positionCaseInsensitiveUTF8 positionUTF8 pow -printf proportionsZTest protocol queryID From 27ee4dd61168795623fd07fafe240c85fa8b5e5a Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 20 Aug 2024 14:17:05 -0300 Subject: [PATCH 091/148] throw syntax error instead of bad arguments in case of add not identified --- src/Parsers/Access/ParserCreateUserQuery.cpp | 26 ++++++++++++------- ..._multiple_authentication_methods.reference | 2 ++ .../03174_multiple_authentication_methods.sh | 3 +++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 0ef658d2377..cbd0612d246 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -223,14 +223,6 @@ namespace { return IParserBase::wrapParseImpl(pos, [&] { - if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) - { - authentication_methods.emplace_back(std::make_shared()); - authentication_methods.back()->type = AuthenticationType::NO_PASSWORD; - - return true; - } - if (!ParserKeyword{Keyword::IDENTIFIED}.ignore(pos, expected)) return false; @@ -269,6 +261,22 @@ namespace }); } + bool parseIdentifiedOrNotIdentified(IParserBase::Pos & pos, Expected & expected, std::vector> & authentication_methods) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (ParserKeyword{Keyword::NOT_IDENTIFIED}.ignore(pos, expected)) + { + authentication_methods.emplace_back(std::make_shared()); + authentication_methods.back()->type = AuthenticationType::NO_PASSWORD; + + return true; + } + + return parseIdentifiedWith(pos, expected, authentication_methods); + }); + } + bool parseHostsWithoutPrefix(IParserBase::Pos & pos, Expected & expected, AllowedClientHosts & hosts) { @@ -545,7 +553,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { if (auth_data.empty()) { - parsed_identified_with = parseIdentifiedWith(pos, expected, auth_data); + parsed_identified_with = parseIdentifiedOrNotIdentified(pos, expected, auth_data); if (!parsed_identified_with) { diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index 8cdc1f9d613..cd8dde2a8a4 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -53,3 +53,5 @@ First auth method can't specify type if WITH keyword is not present SYNTAX_ERROR RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement BAD_ARGUMENTS +ADD NOT IDENTIFIED should result in syntax error +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 5e4c5048d55..e25bbbabb50 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -147,4 +147,7 @@ ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED plaintext_password echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" +echo "ADD NOT IDENTIFIED should result in syntax error" +${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" From 1980959c8b032841763acff7d5ad9757cb636e10 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 20 Aug 2024 14:27:53 -0300 Subject: [PATCH 092/148] exception message --- src/Parsers/Access/ParserCreateUserQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index cbd0612d246..b56dda79875 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -650,12 +650,12 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (has_no_password_authentication_method && auth_data.size() > 1) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "NO_PASSWORD Authentication method cannot co-exist with other authentication methods"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods."); } if (has_no_password_authentication_method && parsed_add_identified_with) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The authentication method 'no_password' cannot be used with the ADD keyword. " + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods. " "Use 'ALTER USER xyz IDENTIFIED WITH no_password' to replace existing authentication methods"); } From f0223aeddef16c041d16f3548dc7573e79fa2ced Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 20 Aug 2024 16:28:12 -0300 Subject: [PATCH 093/148] add on cluster tests --- ..._multiple_authentication_methods.reference | 75 ++++++++ .../03174_multiple_authentication_methods.sh | 166 +++++++++--------- 2 files changed, 162 insertions(+), 79 deletions(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index cd8dde2a8a4..f45d371f156 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -55,3 +55,78 @@ RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement BAD_ARGUMENTS ADD NOT IDENTIFIED should result in syntax error SYNTAX_ERROR +localhost 9000 0 0 0 +localhost 9000 0 0 0 +Basic authentication after user creation +1 +localhost 9000 0 0 0 +Changed password, old password should not work +AUTHENTICATION_FAILED +New password should work +1 +localhost 9000 0 0 0 +Two new passwords were added, should both work +1 +1 +localhost 9000 0 0 0 +Authenticating with ssh key +1 +Altering credentials and keeping only bcrypt_password +localhost 9000 0 0 0 +Asserting SSH does not work anymore +AUTHENTICATION_FAILED +Asserting bcrypt_password works +1 +Adding new bcrypt_password +localhost 9000 0 0 0 +Both current authentication methods should work +1 +1 +Reset authentication methods to new +localhost 9000 0 0 0 +Only the latest should work, below should fail +AUTHENTICATION_FAILED +Should work +1 +Multiple identified with, not allowed +Syntax error +localhost 9000 0 0 0 +CREATE Multiple identified with, not allowed +Syntax error +localhost 9000 0 0 0 +Create user with no identification +localhost 9000 0 0 0 +Add identified with +localhost 9000 0 0 0 +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password +Try to provide no_password mixed with other authentication methods, should not be allowed +BAD_ARGUMENTS +Adding no_password, should fail +BAD_ARGUMENTS +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password +Replacing existing authentication methods in favor of no_password, should succeed +localhost 9000 0 0 0 +CREATE USER u01_03174 IDENTIFIED WITH no_password +Trying to auth with no pwd, should succeed +1 +localhost 9000 0 0 0 +Create user with mix both implicit and explicit auth type, starting with with +localhost 9000 0 0 0 +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +localhost 9000 0 0 0 +Create user with mix both implicit and explicit auth type, starting with by +localhost 9000 0 0 0 +CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password +localhost 9000 0 0 0 +Use WITH without providing authentication type, should fail +Syntax error +Create user with ADD identification, should fail, add is not allowed for create query +BAD_ARGUMENTS +Trailing comma should result in syntax error +SYNTAX_ERROR +First auth method can't specify type if WITH keyword is not present +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement +BAD_ARGUMENTS +ADD NOT IDENTIFIED should result in syntax error +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index e25bbbabb50..4c9914b0cda 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -28,126 +28,134 @@ function test_login_pwd_expect_error test_login_pwd "$1" "$2" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' } -user="u01_03174" +function test +{ + user="u01_03174" -${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY '1'" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password BY '1'" -echo "Basic authentication after user creation" -test_login_pwd ${user} '1' + echo "Basic authentication after user creation" + test_login_pwd ${user} '1' -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password BY '2'" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password BY '2'" -echo "Changed password, old password should not work" -test_login_pwd_expect_error ${user} '1' + echo "Changed password, old password should not work" + test_login_pwd_expect_error ${user} '1' -echo "New password should work" -test_login_pwd ${user} '2' + echo "New password should work" + test_login_pwd ${user} '2' -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password BY '3', plaintext_password BY '4'" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password BY '3', plaintext_password BY '4'" -echo "Two new passwords were added, should both work" -test_login_pwd ${user} '3' + echo "Two new passwords were added, should both work" + test_login_pwd ${user} '3' -test_login_pwd ${user} '4' + test_login_pwd ${user} '4' -ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" + ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" -echo ${ssh_key} > ssh_key + echo ${ssh_key} > ssh_key -echo "Authenticating with ssh key" -${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" + echo "Authenticating with ssh key" + ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" -echo "Altering credentials and keeping only bcrypt_password" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH bcrypt_password BY '5'" + echo "Altering credentials and keeping only bcrypt_password" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH bcrypt_password BY '5'" -echo "Asserting SSH does not work anymore" -${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' + echo "Asserting SSH does not work anymore" + ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' -echo "Asserting bcrypt_password works" -test_login_pwd ${user} '5' + echo "Asserting bcrypt_password works" + test_login_pwd ${user} '5' -echo "Adding new bcrypt_password" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH bcrypt_password BY '6'" + echo "Adding new bcrypt_password" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH bcrypt_password BY '6'" -echo "Both current authentication methods should work" -test_login_pwd ${user} '5' -test_login_pwd ${user} '6' + echo "Both current authentication methods should work" + test_login_pwd ${user} '5' + test_login_pwd ${user} '6' -echo "Reset authentication methods to new" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} RESET AUTHENTICATION METHODS TO NEW" + echo "Reset authentication methods to new" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" -echo "Only the latest should work, below should fail" -test_login_pwd_expect_error ${user} '5' + echo "Only the latest should work, below should fail" + test_login_pwd_expect_error ${user} '5' -echo "Should work" -test_login_pwd ${user} '6' + echo "Should work" + test_login_pwd ${user} '6' -echo "Multiple identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" + echo "Multiple identified with, not allowed" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" -${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER ${user} $1" -echo "CREATE Multiple identified with, not allowed" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" + echo "CREATE Multiple identified with, not allowed" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" -${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" -echo "Create user with no identification" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user}" + echo "Create user with no identification" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1" -echo "Add identified with" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '7'" + echo "Add identified with" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" -echo "Try to provide no_password mixed with other authentication methods, should not be allowed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + echo "Try to provide no_password mixed with other authentication methods, should not be allowed" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "Adding no_password, should fail" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + echo "Adding no_password, should fail" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" -echo "Replacing existing authentication methods in favor of no_password, should succeed" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} IDENTIFIED WITH no_password" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + echo "Replacing existing authentication methods in favor of no_password, should succeed" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH no_password" + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" -echo "Trying to auth with no pwd, should succeed" -test_login_no_pwd ${user} + echo "Trying to auth with no pwd, should succeed" + test_login_no_pwd ${user} -${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" -echo "Create user with mix both implicit and explicit auth type, starting with with" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4';" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + echo "Create user with mix both implicit and explicit auth type, starting with with" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4';" + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" -${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" -echo "Create user with mix both implicit and explicit auth type, starting with by" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4';" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + echo "Create user with mix both implicit and explicit auth type, starting with by" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4';" + ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" -${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" -echo "Use WITH without providing authentication type, should fail" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" + echo "Use WITH without providing authentication type, should fail" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" -echo "Create user with ADD identification, should fail, add is not allowed for create query" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + echo "Create user with ADD identification, should fail, add is not allowed for create query" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "Trailing comma should result in syntax error" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" + echo "Trailing comma should result in syntax error" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" -echo "First auth method can't specify type if WITH keyword is not present" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + echo "First auth method can't specify type if WITH keyword is not present" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" -echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" -${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" -echo "ADD NOT IDENTIFIED should result in syntax error" -${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + echo "ADD NOT IDENTIFIED should result in syntax error" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" -${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + +} + +test "" + +test "ON CLUSTER test_shard_localhost" From 4fd19c3ad993398b43977229ce629d6fa85c06cb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 20 Aug 2024 17:16:12 -0300 Subject: [PATCH 094/148] do not allow reset auth to new to be used along with add identified clauses --- src/Parsers/Access/ParserCreateUserQuery.cpp | 5 +++++ .../03174_multiple_authentication_methods.reference | 4 ++++ .../0_stateless/03174_multiple_authentication_methods.sh | 3 +++ 3 files changed, 12 insertions(+) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index b56dda79875..abf01182ec8 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -660,6 +660,11 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec } + if (reset_authentication_methods_to_new && !auth_data.empty()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses"); + } + if (!alter && !hosts) { String common_host_pattern; diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index f45d371f156..429b04f3907 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -55,6 +55,8 @@ RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement BAD_ARGUMENTS ADD NOT IDENTIFIED should result in syntax error SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses +BAD_ARGUMENTS localhost 9000 0 0 0 localhost 9000 0 0 0 Basic authentication after user creation @@ -130,3 +132,5 @@ RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement BAD_ARGUMENTS ADD NOT IDENTIFIED should result in syntax error SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses +BAD_ARGUMENTS diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 4c9914b0cda..7fe91f409ac 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -152,6 +152,9 @@ function test echo "ADD NOT IDENTIFIED should result in syntax error" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + echo "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" } From d7d40db0365109c55cd9f4eedc1791a6628ba0c2 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 21 Aug 2024 19:07:54 -0300 Subject: [PATCH 095/148] make sure all alters except id related are allowed even if state of user is invalid afterwards --- .../Access/InterpreterCreateUserQuery.cpp | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 5178ad184d6..50d627bfe85 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -82,19 +82,6 @@ namespace user.authentication_methods.clear(); } - auto number_of_authentication_methods = user.authentication_methods.size() + authentication_methods.size(); - if (number_of_authentication_methods > max_number_of_authentication_methods) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user." - "Check the `max_authentication_methods_per_user` setting"); - } - - for (const auto & authentication_method : authentication_methods) - { - user.authentication_methods.emplace_back(authentication_method); - } - // drop existing ones and keep the most recent if (reset_authentication_methods) { @@ -103,6 +90,24 @@ namespace user.authentication_methods.emplace_back(backup_authentication_method); } + if (!authentication_methods.empty()) + { + // we only check if user exceeds the allowed quantity of authentication methods in case the create/alter query includes + // authentication information. Otherwise, we can bypass this check to avoid blocking non-authentication related alters. + auto number_of_authentication_methods = user.authentication_methods.size() + authentication_methods.size(); + if (number_of_authentication_methods > max_number_of_authentication_methods) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user." + "Check the `max_authentication_methods_per_user` setting"); + } + } + + for (const auto & authentication_method : authentication_methods) + { + user.authentication_methods.emplace_back(authentication_method); + } + if (!query.alter) { for (const auto & authentication_method : user.authentication_methods) From 13d5b029a408a62d05fa27a6a211da5424540024 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 22 Aug 2024 09:41:10 -0300 Subject: [PATCH 096/148] make max_number_of_authentication_methods=0 unlimited --- src/Access/AccessEntityIO.cpp | 2 +- src/Core/ServerSettings.h | 2 +- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Access/AccessEntityIO.cpp b/src/Access/AccessEntityIO.cpp index 53b073cb750..cc1b7eee807 100644 --- a/src/Access/AccessEntityIO.cpp +++ b/src/Access/AccessEntityIO.cpp @@ -82,7 +82,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition) if (res) throw Exception(ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION, "Two access entities attached in the same file"); res = user = std::make_unique(); - InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true, /* max_number_of_authentication_methods = */ std::numeric_limits::max()); + InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true, /* max_number_of_authentication_methods = zero is unlimited*/ 0); } else if (auto * create_role_query = query->as()) { diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index ee5eefddc83..dab6401157d 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -116,7 +116,7 @@ namespace DB M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \ M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \ M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \ - M(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users.", 0) \ + M(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users. Zero means unlimited", 0) \ M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \ M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \ \ diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 50d627bfe85..467de1e025c 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -90,7 +90,8 @@ namespace user.authentication_methods.emplace_back(backup_authentication_method); } - if (!authentication_methods.empty()) + // max_number_of_authentication_methods == 0 means unlimited + if (!authentication_methods.empty() && max_number_of_authentication_methods != 0) { // we only check if user exceeds the allowed quantity of authentication methods in case the create/alter query includes // authentication information. Otherwise, we can bypass this check to avoid blocking non-authentication related alters. From 91cceccb4b7a6e7b93c7fa41a777a64015271697 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 22 Aug 2024 09:48:53 -0300 Subject: [PATCH 097/148] add setting docs --- .../server-configuration-parameters/settings.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 9fce83a0dc4..8a2cdeb3216 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -3121,3 +3121,15 @@ Default value: "default" **See Also** - [Workload Scheduling](/docs/en/operations/workload-scheduling.md) + +## max_authentication_methods_per_user {#max_authentication_methods_per_user} + +The maximum number of authentication methods a user can be created with or altered to. +Changing this setting does not affect existing users. Create/alter authentication related queries will fail if they exceed the limit specified in this setting. +Non authentication create/alter queries will succeed. + +Type: UInt64 + +Default value: 100 + +Zero means unlimited From 62ce2999ae43cc9bf2f27f762ed8202ebe5e2062 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 22 Aug 2024 10:00:20 -0300 Subject: [PATCH 098/148] minor grammar --- docs/en/operations/server-configuration-parameters/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 8a2cdeb3216..ff41f501689 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -3125,7 +3125,7 @@ Default value: "default" ## max_authentication_methods_per_user {#max_authentication_methods_per_user} The maximum number of authentication methods a user can be created with or altered to. -Changing this setting does not affect existing users. Create/alter authentication related queries will fail if they exceed the limit specified in this setting. +Changing this setting does not affect existing users. Create/alter authentication-related queries will fail if they exceed the limit specified in this setting. Non authentication create/alter queries will succeed. Type: UInt64 From cb7ef910e2f3a165c3ab48610fac9c9e1059d7b6 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 23 Aug 2024 13:50:38 -0300 Subject: [PATCH 099/148] Update 03174_multiple_authentication_methods.sh --- .../0_stateless/03174_multiple_authentication_methods.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 7fe91f409ac..310187c62fc 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-parallel +# Tags: no-fasttest, no-parallel, no-replicated-database +# Tag no-replicated-database: https://s3.amazonaws.com/clickhouse-test-reports/65277/43e9a7ba4bbf7f20145531b384a31304895b55bc/stateless_tests__release__old_analyzer__s3__databasereplicated__[1_2].html and https://github.com/ClickHouse/ClickHouse/blob/011c694117845500c82f9563c65930429979982f/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh#L4 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From a65d175a8137232996ab3adb8124e87e2157d3ba Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 27 Aug 2024 10:40:04 -0300 Subject: [PATCH 100/148] change parsing logic a bit --- .../Access/InterpreterCreateUserQuery.cpp | 23 +++--- src/Parsers/Access/ParserCreateUserQuery.cpp | 77 ++++++++----------- ..._multiple_authentication_methods.reference | 32 ++++---- .../03174_multiple_authentication_methods.sh | 21 +++-- 4 files changed, 68 insertions(+), 85 deletions(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 467de1e025c..e0d99b64dfb 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -67,17 +67,8 @@ namespace user.authentication_methods.emplace_back(); } - bool has_no_password_authentication_method = std::find_if( - user.authentication_methods.begin(), - user.authentication_methods.end(), - [](const AuthenticationData & authentication_method) - { - return authentication_method.getType() == AuthenticationType::NO_PASSWORD; - }) != user.authentication_methods.end(); - // 1. an IDENTIFIED WITH will drop existing authentication methods in favor of new ones. - // 2. if the user contains an auth method of type NO_PASSWORD and another one is being added, NO_PASSWORD must be dropped - if (replace_authentication_methods || (has_no_password_authentication_method && !authentication_methods.empty())) + if (replace_authentication_methods) { user.authentication_methods.clear(); } @@ -109,6 +100,18 @@ namespace user.authentication_methods.emplace_back(authentication_method); } + bool has_no_password_authentication_method = std::find_if(user.authentication_methods.begin(), + user.authentication_methods.end(), + [](const AuthenticationData & authentication_data) + { + return authentication_data.getType() == AuthenticationType::NO_PASSWORD; + }) != user.authentication_methods.end(); + + if (has_no_password_authentication_method && user.authentication_methods.size() > 1) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods."); + } + if (!query.alter) { for (const auto & authentication_method : user.authentication_methods) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index abf01182ec8..593026b19ad 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -53,7 +53,8 @@ namespace Expected & expected, std::shared_ptr & auth_data, bool is_type_specifier_mandatory, - bool is_type_specifier_allowed) + bool is_type_specifier_allowed, + bool should_parse_no_password) { return IParserBase::wrapParseImpl(pos, [&] { @@ -67,7 +68,7 @@ namespace bool expect_public_ssh_key = false; bool expect_http_auth_server = false; - for (auto check_type : collections::range(AuthenticationType::MAX)) + auto parse_non_password_based_type = [&](auto check_type) { if (ParserKeyword{AuthenticationTypeInfo::get(check_type).keyword}.ignore(pos, expected)) { @@ -86,7 +87,20 @@ namespace else if (check_type != AuthenticationType::NO_PASSWORD) expect_password = true; - break; + return true; + } + + return false; + }; + + { + const auto first_authentication_type_element_to_check + = should_parse_no_password ? AuthenticationType::NO_PASSWORD : AuthenticationType::PLAINTEXT_PASSWORD; + + for (auto check_type : collections::range(first_authentication_type_element_to_check, AuthenticationType::MAX)) + { + if (parse_non_password_based_type(check_type)) + break; } } @@ -219,7 +233,11 @@ namespace } - bool parseIdentifiedWith(IParserBase::Pos & pos, Expected & expected, std::vector> & authentication_methods) + bool parseIdentifiedWith( + IParserBase::Pos & pos, + Expected & expected, + std::vector> & authentication_methods, + bool should_parse_no_password) { return IParserBase::wrapParseImpl(pos, [&] { @@ -232,7 +250,7 @@ namespace std::shared_ptr ast_authentication_data; - if (!parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory, is_type_specifier_mandatory)) + if (!parseAuthenticationData(pos, expected, ast_authentication_data, is_type_specifier_mandatory, is_type_specifier_mandatory, should_parse_no_password)) { return false; } @@ -248,7 +266,7 @@ namespace { std::shared_ptr ast_authentication_data; - if (!parseAuthenticationData(aux_pos, expected, ast_authentication_data, false, true)) + if (!parseAuthenticationData(aux_pos, expected, ast_authentication_data, false, true, should_parse_no_password)) { break; } @@ -273,7 +291,7 @@ namespace return true; } - return parseIdentifiedWith(pos, expected, authentication_methods); + return parseIdentifiedWith(pos, expected, authentication_methods, true); }); } @@ -480,7 +498,7 @@ namespace return false; } - return parseIdentifiedWith(pos, expected, auth_data); + return parseIdentifiedWith(pos, expected, auth_data, false); }); } @@ -551,29 +569,19 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec while (true) { - if (auth_data.empty()) + if (auth_data.empty() && !reset_authentication_methods_to_new) { parsed_identified_with = parseIdentifiedOrNotIdentified(pos, expected, auth_data); - if (!parsed_identified_with) + if (!parsed_identified_with && alter) { parsed_add_identified_with = parseAddIdentifiedWith(pos, expected, auth_data); - - if (parsed_add_identified_with && !alter) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Create user query is not allowed to have ADD IDENTIFIED, remove the ADD keyword."); - } } } - if (!reset_authentication_methods_to_new) + if (!reset_authentication_methods_to_new && alter && auth_data.empty()) { reset_authentication_methods_to_new = parseResetAuthenticationMethods(pos, expected); - - if (reset_authentication_methods_to_new && !alter) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "RESET AUTHENTICATION METHODS TO NEW can only be used on ALTER statement"); - } } if (!valid_until) @@ -640,31 +648,6 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec break; } - bool has_no_password_authentication_method = std::find_if( - auth_data.begin(), - auth_data.end(), - [](const std::shared_ptr & ast_authentication_data) - { - return ast_authentication_data->type == AuthenticationType::NO_PASSWORD; - }) != auth_data.end(); - - if (has_no_password_authentication_method && auth_data.size() > 1) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods."); - } - - if (has_no_password_authentication_method && parsed_add_identified_with) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods. " - "Use 'ALTER USER xyz IDENTIFIED WITH no_password' to replace existing authentication methods"); - - } - - if (reset_authentication_methods_to_new && !auth_data.empty()) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses"); - } - if (!alter && !hosts) { String common_host_pattern; @@ -700,7 +683,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->storage_name = std::move(storage_name); query->reset_authentication_methods_to_new = reset_authentication_methods_to_new; query->add_identified_with = parsed_add_identified_with; - query->replace_authentication_methods = parsed_identified_with || has_no_password_authentication_method; + query->replace_authentication_methods = parsed_identified_with; for (const auto & authentication_method : query->authentication_methods) { diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index 429b04f3907..c7dbf67d784 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -28,13 +28,12 @@ Syntax error CREATE Multiple identified with, not allowed Syntax error Create user with no identification -Add identified with -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password -Try to provide no_password mixed with other authentication methods, should not be allowed +Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types BAD_ARGUMENTS +Try to add no_password mixed with other authentication methods, should not be allowed +SYNTAX_ERROR Adding no_password, should fail -BAD_ARGUMENTS -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password +SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed @@ -46,17 +45,18 @@ CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcryp Use WITH without providing authentication type, should fail Syntax error Create user with ADD identification, should fail, add is not allowed for create query -BAD_ARGUMENTS +SYNTAX_ERROR Trailing comma should result in syntax error SYNTAX_ERROR First auth method can't specify type if WITH keyword is not present SYNTAX_ERROR RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement -BAD_ARGUMENTS +SYNTAX_ERROR ADD NOT IDENTIFIED should result in syntax error SYNTAX_ERROR RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses -BAD_ARGUMENTS +SYNTAX_ERROR +On cluster tests localhost 9000 0 0 0 localhost 9000 0 0 0 Basic authentication after user creation @@ -98,14 +98,12 @@ Syntax error localhost 9000 0 0 0 Create user with no identification localhost 9000 0 0 0 -Add identified with -localhost 9000 0 0 0 -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password -Try to provide no_password mixed with other authentication methods, should not be allowed +Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types BAD_ARGUMENTS +Try to add no_password mixed with other authentication methods, should not be allowed +SYNTAX_ERROR Adding no_password, should fail -BAD_ARGUMENTS -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password +SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed localhost 9000 0 0 0 CREATE USER u01_03174 IDENTIFIED WITH no_password @@ -123,14 +121,14 @@ localhost 9000 0 0 0 Use WITH without providing authentication type, should fail Syntax error Create user with ADD identification, should fail, add is not allowed for create query -BAD_ARGUMENTS +SYNTAX_ERROR Trailing comma should result in syntax error SYNTAX_ERROR First auth method can't specify type if WITH keyword is not present SYNTAX_ERROR RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement -BAD_ARGUMENTS +SYNTAX_ERROR ADD NOT IDENTIFIED should result in syntax error SYNTAX_ERROR RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses -BAD_ARGUMENTS +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 310187c62fc..bfb7775dbb0 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -102,17 +102,14 @@ function test echo "Create user with no identification" ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1" - echo "Add identified with" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" + echo "Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" - ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" - - echo "Try to provide no_password mixed with other authentication methods, should not be allowed" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + echo "Try to add no_password mixed with other authentication methods, should not be allowed" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" echo "Adding no_password, should fail" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" - ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" echo "Replacing existing authentication methods in favor of no_password, should succeed" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH no_password" @@ -139,7 +136,7 @@ function test ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" echo "Create user with ADD identification, should fail, add is not allowed for create query" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" echo "Trailing comma should result in syntax error" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" @@ -148,13 +145,13 @@ function test ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" echo "ADD NOT IDENTIFIED should result in syntax error" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" echo "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" @@ -162,4 +159,6 @@ function test test "" +echo "On cluster tests" + test "ON CLUSTER test_shard_localhost" From a0ab22e031f6866826fe6cd85a00c4eed8ee9359 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 27 Aug 2024 13:01:10 -0300 Subject: [PATCH 101/148] style --- src/Parsers/Access/ParserCreateUserQuery.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index 593026b19ad..8bfc84a28a6 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -25,11 +25,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - namespace { bool parseRenameTo(IParserBase::Pos & pos, Expected & expected, std::optional & new_name) From 0b29aef1a07983579bc454e0941a43c7954827e9 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 27 Aug 2024 15:25:41 -0300 Subject: [PATCH 102/148] remove extra dot from ex message --- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index e0d99b64dfb..86d052ab370 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -109,7 +109,7 @@ namespace if (has_no_password_authentication_method && user.authentication_methods.size() > 1) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods."); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Authentication method 'no_password' cannot co-exist with other authentication methods"); } if (!query.alter) From c298b20ba9e187a1ef9a5cbdacf121876cae334d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 27 Aug 2024 16:17:35 -0300 Subject: [PATCH 103/148] extract some tests into sql as an attempt to make them run faster for flaky check.. --- ..._multiple_authentication_methods.reference | 66 ----------------- .../03174_multiple_authentication_methods.sh | 58 --------------- ...ltiple_authentication_methods_no_login.sql | 70 +++++++++++++++++++ 3 files changed, 70 insertions(+), 124 deletions(-) create mode 100644 tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index c7dbf67d784..cc8628c1ac0 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -23,39 +23,10 @@ Only the latest should work, below should fail AUTHENTICATION_FAILED Should work 1 -Multiple identified with, not allowed -Syntax error -CREATE Multiple identified with, not allowed -Syntax error -Create user with no identification -Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types -BAD_ARGUMENTS -Try to add no_password mixed with other authentication methods, should not be allowed -SYNTAX_ERROR -Adding no_password, should fail -SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 -Create user with mix both implicit and explicit auth type, starting with with -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password -Create user with mix both implicit and explicit auth type, starting with by -CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password -Use WITH without providing authentication type, should fail -Syntax error -Create user with ADD identification, should fail, add is not allowed for create query -SYNTAX_ERROR -Trailing comma should result in syntax error -SYNTAX_ERROR -First auth method can't specify type if WITH keyword is not present -SYNTAX_ERROR -RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement -SYNTAX_ERROR -ADD NOT IDENTIFIED should result in syntax error -SYNTAX_ERROR -RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses -SYNTAX_ERROR On cluster tests localhost 9000 0 0 0 localhost 9000 0 0 0 @@ -90,45 +61,8 @@ Only the latest should work, below should fail AUTHENTICATION_FAILED Should work 1 -Multiple identified with, not allowed -Syntax error -localhost 9000 0 0 0 -CREATE Multiple identified with, not allowed -Syntax error -localhost 9000 0 0 0 -Create user with no identification -localhost 9000 0 0 0 -Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types -BAD_ARGUMENTS -Try to add no_password mixed with other authentication methods, should not be allowed -SYNTAX_ERROR -Adding no_password, should fail -SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed localhost 9000 0 0 0 CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 -localhost 9000 0 0 0 -Create user with mix both implicit and explicit auth type, starting with with -localhost 9000 0 0 0 -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password -localhost 9000 0 0 0 -Create user with mix both implicit and explicit auth type, starting with by -localhost 9000 0 0 0 -CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password -localhost 9000 0 0 0 -Use WITH without providing authentication type, should fail -Syntax error -Create user with ADD identification, should fail, add is not allowed for create query -SYNTAX_ERROR -Trailing comma should result in syntax error -SYNTAX_ERROR -First auth method can't specify type if WITH keyword is not present -SYNTAX_ERROR -RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement -SYNTAX_ERROR -ADD NOT IDENTIFIED should result in syntax error -SYNTAX_ERROR -RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses -SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index bfb7775dbb0..d775034bd59 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -89,28 +89,6 @@ function test echo "Should work" test_login_pwd ${user} '6' - echo "Multiple identified with, not allowed" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" - - ${CLICKHOUSE_CLIENT} --query "DROP USER ${user} $1" - - echo "CREATE Multiple identified with, not allowed" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" - - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" - - echo "Create user with no identification" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1" - - echo "Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" - - echo "Try to add no_password mixed with other authentication methods, should not be allowed" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - - echo "Adding no_password, should fail" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - echo "Replacing existing authentication methods in favor of no_password, should succeed" ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH no_password" ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" @@ -118,43 +96,7 @@ function test echo "Trying to auth with no pwd, should succeed" test_login_no_pwd ${user} - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" - - echo "Create user with mix both implicit and explicit auth type, starting with with" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4';" - ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" - - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" - - echo "Create user with mix both implicit and explicit auth type, starting with by" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4';" - ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" - - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" - - echo "Use WITH without providing authentication type, should fail" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" - - echo "Create user with ADD identification, should fail, add is not allowed for create query" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - - echo "Trailing comma should result in syntax error" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" - - echo "First auth method can't specify type if WITH keyword is not present" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - - echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - - echo "ADD NOT IDENTIFIED should result in syntax error" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - - echo "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" - } test "" diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql new file mode 100644 index 00000000000..b21b90700b6 --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql @@ -0,0 +1,70 @@ +DROP USER IF EXISTS u_03174_no_login; + +CREATE USER u_03174_no_login; + +-- multiple identified with, not allowed +ALTER USER u_03174_no_login IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'; -- { clientError SYNTAX_ERROR } + +-- CREATE Multiple identified with, not allowed +CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'; -- { clientError SYNTAX_ERROR } + +DROP USER u_03174_no_login; + +-- Create user with no identification +CREATE USER u_03174_no_login; + +-- Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types +ALTER USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '7'; -- { serverError BAD_ARGUMENTS } + +-- Try to add no_password mixed with other authentication methods, should not be allowed +ALTER USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '8', no_password; -- { clientError SYNTAX_ERROR } + +-- Adding no_password, should fail +ALTER USER u_03174_no_login ADD IDENTIFIED WITH no_password; -- { clientError SYNTAX_ERROR } + +DROP USER IF EXISTS u_03174_no_login; + +-- Create user with mix both implicit and explicit auth type, starting with with +CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_no_login; + +DROP USER IF EXISTS u_03174_no_login; + +-- Create user with mix both implicit and explicit auth type, starting with with. On cluster +CREATE USER u_03174_no_login ON CLUSTER test_shard_localhost IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_no_login; + +DROP USER IF EXISTS u_03174_no_login; + +-- Create user with mix both implicit and explicit auth type, starting with by +CREATE USER u_03174_no_login IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_no_login; + +DROP USER IF EXISTS u_03174_no_login; + +-- Create user with mix both implicit and explicit auth type, starting with by. On cluster +CREATE USER u_03174_no_login ON CLUSTER test_shard_localhost IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_no_login; + +DROP USER IF EXISTS u_03174_no_login; + +-- Use WITH without providing authentication type, should fail +CREATE USER u_03174_no_login IDENTIFIED WITH BY '1'; -- { clientError SYNTAX_ERROR } + +-- Create user with ADD identification, should fail, add is not allowed for create query +CREATE USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '1'; -- { clientError SYNTAX_ERROR } + +-- Trailing comma should result in syntax error +ALTER USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '1',; -- { clientError SYNTAX_ERROR } + +-- First auth method can't specify type if WITH keyword is not present +CREATE USER u_03174_no_login IDENTIFIED plaintext_password by '1'; -- { clientError SYNTAX_ERROR } + +-- RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement +CREATE USER u_03174_no_login RESET AUTHENTICATION METHODS TO NEW; -- { clientError SYNTAX_ERROR } + +-- ADD NOT IDENTIFIED should result in syntax error +ALTER USER u_03174_no_login ADD NOT IDENTIFIED; -- { clientError SYNTAX_ERROR } + +-- RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses +ALTER USER u_03174_no_login IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW; -- { clientError SYNTAX_ERROR } From 50b3d3172cc2c0ed09883bcaa13a6824b8875b30 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 27 Aug 2024 16:48:10 -0300 Subject: [PATCH 104/148] add no fast test --- ...03174_multiple_authentication_methods_no_login.reference | 6 ++++++ .../03174_multiple_authentication_methods_no_login.sql | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference new file mode 100644 index 00000000000..f451567b666 --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference @@ -0,0 +1,6 @@ +CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +localhost 9000 0 0 0 +CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +CREATE USER u_03174_no_login IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password +localhost 9000 0 0 0 +CREATE USER u_03174_no_login IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql index b21b90700b6..a79c13b94be 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + DROP USER IF EXISTS u_03174_no_login; CREATE USER u_03174_no_login; From 41a4a97ca3c47bc4420b9247572f937569b98e6f Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 27 Aug 2024 19:57:59 -0300 Subject: [PATCH 105/148] no parallel --- .../03174_multiple_authentication_methods_no_login.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql index a79c13b94be..5a158b339ec 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-parallel DROP USER IF EXISTS u_03174_no_login; From ae8d90f6b81a79bb08f9314e61fd24732ba51a30 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 09:05:25 -0300 Subject: [PATCH 106/148] no replicated database --- .../03174_multiple_authentication_methods_no_login.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql index 5a158b339ec..124241bf6fd 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql @@ -1,4 +1,5 @@ --- Tags: no-fasttest, no-parallel +-- Tags: no-fasttest, no-parallel, no-replicated-database +-- Tag no-replicated-database: https://s3.amazonaws.com/clickhouse-test-reports/65277/43e9a7ba4bbf7f20145531b384a31304895b55bc/stateless_tests__release__old_analyzer__s3__databasereplicated__[1_2].html and https://github.com/ClickHouse/ClickHouse/blob/011c694117845500c82f9563c65930429979982f/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh#L4 DROP USER IF EXISTS u_03174_no_login; From b29e00b83830e238a45b64a42bf135d9c3614b54 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 09:17:04 -0300 Subject: [PATCH 107/148] add space --- src/Interpreters/Access/InterpreterCreateUserQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp index 86d052ab370..81600b2b6eb 100644 --- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp @@ -90,7 +90,7 @@ namespace if (number_of_authentication_methods > max_number_of_authentication_methods) { throw Exception(ErrorCodes::BAD_ARGUMENTS, - "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user." + "User can not be created/updated because it exceeds the allowed quantity of authentication methods per user. " "Check the `max_authentication_methods_per_user` setting"); } } From 9f7f6c6f931c800fc8126bf0b94b59137638226d Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 12:54:22 -0300 Subject: [PATCH 108/148] use curl client for create/drop/alter queries to make test run faster --- .../03174_multiple_authentication_methods.sh | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index d775034bd59..97c20fe6e16 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -33,14 +33,14 @@ function test { user="u01_03174" - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user} $1" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" - ${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password BY '1'" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password BY '1'" echo "Basic authentication after user creation" test_login_pwd ${user} '1' - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password BY '2'" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password BY '2'" echo "Changed password, old password should not work" test_login_pwd_expect_error ${user} '1' @@ -48,7 +48,7 @@ function test echo "New password should work" test_login_pwd ${user} '2' - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password BY '3', plaintext_password BY '4'" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password BY '3', plaintext_password BY '4'" echo "Two new passwords were added, should both work" test_login_pwd ${user} '3' @@ -57,7 +57,7 @@ function test ssh_pub_key="AAAAC3NzaC1lZDI1NTE5AAAAIBzqa3duS0ce6QYkzUgko9W0Ux7i7d3xPoseFrwnhY4Y" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH ssh_key BY KEY '${ssh_pub_key}' TYPE 'ssh-ed25519'" echo ${ssh_key} > ssh_key @@ -65,7 +65,7 @@ function test ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" echo "Altering credentials and keeping only bcrypt_password" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH bcrypt_password BY '5'" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH bcrypt_password BY '5'" echo "Asserting SSH does not work anymore" ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' @@ -74,14 +74,14 @@ function test test_login_pwd ${user} '5' echo "Adding new bcrypt_password" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 ADD IDENTIFIED WITH bcrypt_password BY '6'" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH bcrypt_password BY '6'" echo "Both current authentication methods should work" test_login_pwd ${user} '5' test_login_pwd ${user} '6' echo "Reset authentication methods to new" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" echo "Only the latest should work, below should fail" test_login_pwd_expect_error ${user} '5' @@ -90,13 +90,13 @@ function test test_login_pwd ${user} '6' echo "Replacing existing authentication methods in favor of no_password, should succeed" - ${CLICKHOUSE_CLIENT} --query "ALTER USER ${user} $1 IDENTIFIED WITH no_password" - ${CLICKHOUSE_CLIENT} --query "SHOW CREATE USER ${user}" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH no_password" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" echo "Trying to auth with no pwd, should succeed" test_login_no_pwd ${user} - ${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user}" } test "" From 313b6b533f3ab2059ad07fdbdf4b32d396f19e09 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 13:54:31 -0300 Subject: [PATCH 109/148] further optimize test by using http client instead of clickhouse-client --- .../0_stateless/03174_multiple_authentication_methods.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 97c20fe6e16..41c2f89bc77 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -16,12 +16,12 @@ ssh_key="-----BEGIN OPENSSH PRIVATE KEY----- function test_login_no_pwd { - ${CLICKHOUSE_CLIENT} --user $1 --query "select 1" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=$1" -d "select 1" } function test_login_pwd { - ${CLICKHOUSE_CLIENT} --user $1 --password $2 --query "select 1" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=$1&password=$2" -d "select 1" } function test_login_pwd_expect_error From 7879915493723ba1e1cd46603577f490afa41bbb Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 14:01:24 -0300 Subject: [PATCH 110/148] merge test fils now that they are faster --- ..._multiple_authentication_methods.reference | 67 +++++++++++++++++ .../03174_multiple_authentication_methods.sh | 60 ++++++++++++++- ..._authentication_methods_no_login.reference | 6 -- ...ltiple_authentication_methods_no_login.sql | 73 ------------------- 4 files changed, 126 insertions(+), 80 deletions(-) delete mode 100644 tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference delete mode 100644 tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index cc8628c1ac0..759097b3da2 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -23,10 +23,39 @@ Only the latest should work, below should fail AUTHENTICATION_FAILED Should work 1 +Multiple identified with, not allowed +Syntax error +CREATE Multiple identified with, not allowed +Syntax error +Create user with no identification +Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types +BAD_ARGUMENTS +Try to add no_password mixed with other authentication methods, should not be allowed +SYNTAX_ERROR +Adding no_password, should fail +SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 +Create user with mix both implicit and explicit auth type, starting with with +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +Create user with mix both implicit and explicit auth type, starting with by +CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password +Use WITH without providing authentication type, should fail +Syntax error +Create user with ADD identification, should fail, add is not allowed for create query +SYNTAX_ERROR +Trailing comma should result in syntax error +SYNTAX_ERROR +First auth method can't specify type if WITH keyword is not present +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement +SYNTAX_ERROR +ADD NOT IDENTIFIED should result in syntax error +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses +SYNTAX_ERROR On cluster tests localhost 9000 0 0 0 localhost 9000 0 0 0 @@ -61,8 +90,46 @@ Only the latest should work, below should fail AUTHENTICATION_FAILED Should work 1 +Multiple identified with, not allowed +Syntax error +localhost 9000 0 0 0 +CREATE Multiple identified with, not allowed +Syntax error +localhost 9000 0 0 0 +Create user with no identification +localhost 9000 0 0 0 +Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types +BAD_ARGUMENTS +BAD_ARGUMENTS +Try to add no_password mixed with other authentication methods, should not be allowed +SYNTAX_ERROR +Adding no_password, should fail +SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed localhost 9000 0 0 0 CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 +localhost 9000 0 0 0 +Create user with mix both implicit and explicit auth type, starting with with +localhost 9000 0 0 0 +CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +localhost 9000 0 0 0 +Create user with mix both implicit and explicit auth type, starting with by +localhost 9000 0 0 0 +CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password +localhost 9000 0 0 0 +Use WITH without providing authentication type, should fail +Syntax error +Create user with ADD identification, should fail, add is not allowed for create query +SYNTAX_ERROR +Trailing comma should result in syntax error +SYNTAX_ERROR +First auth method can't specify type if WITH keyword is not present +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement +SYNTAX_ERROR +ADD NOT IDENTIFIED should result in syntax error +SYNTAX_ERROR +RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses +SYNTAX_ERROR diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 41c2f89bc77..a7969e55a13 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -89,6 +89,28 @@ function test echo "Should work" test_login_pwd ${user} '6' + echo "Multiple identified with, not allowed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER ${user} $1" + + echo "CREATE Multiple identified with, not allowed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + echo "Create user with no identification" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1" + + echo "Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + + echo "Try to add no_password mixed with other authentication methods, should not be allowed" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + + echo "Adding no_password, should fail" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + echo "Replacing existing authentication methods in favor of no_password, should succeed" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH no_password" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" @@ -96,11 +118,47 @@ function test echo "Trying to auth with no pwd, should succeed" test_login_no_pwd ${user} + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + echo "Create user with mix both implicit and explicit auth type, starting with with" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4';" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + echo "Create user with mix both implicit and explicit auth type, starting with by" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4';" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" + + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" + + echo "Use WITH without providing authentication type, should fail" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" + + echo "Create user with ADD identification, should fail, add is not allowed for create query" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + + echo "Trailing comma should result in syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" + + echo "First auth method can't specify type if WITH keyword is not present" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + + echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + + echo "ADD NOT IDENTIFIED should result in syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + + echo "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user}" + } test "" echo "On cluster tests" -test "ON CLUSTER test_shard_localhost" +test "ON CLUSTER test_shard_localhost" \ No newline at end of file diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference deleted file mode 100644 index f451567b666..00000000000 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.reference +++ /dev/null @@ -1,6 +0,0 @@ -CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password -localhost 9000 0 0 0 -CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password -CREATE USER u_03174_no_login IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password -localhost 9000 0 0 0 -CREATE USER u_03174_no_login IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql deleted file mode 100644 index 124241bf6fd..00000000000 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods_no_login.sql +++ /dev/null @@ -1,73 +0,0 @@ --- Tags: no-fasttest, no-parallel, no-replicated-database --- Tag no-replicated-database: https://s3.amazonaws.com/clickhouse-test-reports/65277/43e9a7ba4bbf7f20145531b384a31304895b55bc/stateless_tests__release__old_analyzer__s3__databasereplicated__[1_2].html and https://github.com/ClickHouse/ClickHouse/blob/011c694117845500c82f9563c65930429979982f/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh#L4 - -DROP USER IF EXISTS u_03174_no_login; - -CREATE USER u_03174_no_login; - --- multiple identified with, not allowed -ALTER USER u_03174_no_login IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'; -- { clientError SYNTAX_ERROR } - --- CREATE Multiple identified with, not allowed -CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'; -- { clientError SYNTAX_ERROR } - -DROP USER u_03174_no_login; - --- Create user with no identification -CREATE USER u_03174_no_login; - --- Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types -ALTER USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '7'; -- { serverError BAD_ARGUMENTS } - --- Try to add no_password mixed with other authentication methods, should not be allowed -ALTER USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '8', no_password; -- { clientError SYNTAX_ERROR } - --- Adding no_password, should fail -ALTER USER u_03174_no_login ADD IDENTIFIED WITH no_password; -- { clientError SYNTAX_ERROR } - -DROP USER IF EXISTS u_03174_no_login; - --- Create user with mix both implicit and explicit auth type, starting with with -CREATE USER u_03174_no_login IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4'; -SHOW CREATE USER u_03174_no_login; - -DROP USER IF EXISTS u_03174_no_login; - --- Create user with mix both implicit and explicit auth type, starting with with. On cluster -CREATE USER u_03174_no_login ON CLUSTER test_shard_localhost IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4'; -SHOW CREATE USER u_03174_no_login; - -DROP USER IF EXISTS u_03174_no_login; - --- Create user with mix both implicit and explicit auth type, starting with by -CREATE USER u_03174_no_login IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4'; -SHOW CREATE USER u_03174_no_login; - -DROP USER IF EXISTS u_03174_no_login; - --- Create user with mix both implicit and explicit auth type, starting with by. On cluster -CREATE USER u_03174_no_login ON CLUSTER test_shard_localhost IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4'; -SHOW CREATE USER u_03174_no_login; - -DROP USER IF EXISTS u_03174_no_login; - --- Use WITH without providing authentication type, should fail -CREATE USER u_03174_no_login IDENTIFIED WITH BY '1'; -- { clientError SYNTAX_ERROR } - --- Create user with ADD identification, should fail, add is not allowed for create query -CREATE USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '1'; -- { clientError SYNTAX_ERROR } - --- Trailing comma should result in syntax error -ALTER USER u_03174_no_login ADD IDENTIFIED WITH plaintext_password by '1',; -- { clientError SYNTAX_ERROR } - --- First auth method can't specify type if WITH keyword is not present -CREATE USER u_03174_no_login IDENTIFIED plaintext_password by '1'; -- { clientError SYNTAX_ERROR } - --- RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement -CREATE USER u_03174_no_login RESET AUTHENTICATION METHODS TO NEW; -- { clientError SYNTAX_ERROR } - --- ADD NOT IDENTIFIED should result in syntax error -ALTER USER u_03174_no_login ADD NOT IDENTIFIED; -- { clientError SYNTAX_ERROR } - --- RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses -ALTER USER u_03174_no_login IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW; -- { clientError SYNTAX_ERROR } From b2795f06dc65d0bb9668f982e6f5db28d886b430 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 14:02:09 -0300 Subject: [PATCH 111/148] add extra line --- .../0_stateless/03174_multiple_authentication_methods.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index a7969e55a13..715b99a4334 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -161,4 +161,4 @@ test "" echo "On cluster tests" -test "ON CLUSTER test_shard_localhost" \ No newline at end of file +test "ON CLUSTER test_shard_localhost" From a690539935c380040afc49fffa16342a97381228 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 18:03:27 -0300 Subject: [PATCH 112/148] fix test --- .../0_stateless/03174_multiple_authentication_methods.reference | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index 759097b3da2..c7dbf67d784 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -100,7 +100,6 @@ Create user with no identification localhost 9000 0 0 0 Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types BAD_ARGUMENTS -BAD_ARGUMENTS Try to add no_password mixed with other authentication methods, should not be allowed SYNTAX_ERROR Adding no_password, should fail From cb6d142947fa4fcd614f298b244c10c79aff0cd0 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Wed, 28 Aug 2024 18:09:30 -0300 Subject: [PATCH 113/148] remove non clustered tests to make it faster :D --- ..._multiple_authentication_methods.reference | 59 ------------------- .../03174_multiple_authentication_methods.sh | 4 -- 2 files changed, 63 deletions(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index c7dbf67d784..d503b78a2bf 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -1,62 +1,3 @@ -Basic authentication after user creation -1 -Changed password, old password should not work -AUTHENTICATION_FAILED -New password should work -1 -Two new passwords were added, should both work -1 -1 -Authenticating with ssh key -1 -Altering credentials and keeping only bcrypt_password -Asserting SSH does not work anymore -AUTHENTICATION_FAILED -Asserting bcrypt_password works -1 -Adding new bcrypt_password -Both current authentication methods should work -1 -1 -Reset authentication methods to new -Only the latest should work, below should fail -AUTHENTICATION_FAILED -Should work -1 -Multiple identified with, not allowed -Syntax error -CREATE Multiple identified with, not allowed -Syntax error -Create user with no identification -Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types -BAD_ARGUMENTS -Try to add no_password mixed with other authentication methods, should not be allowed -SYNTAX_ERROR -Adding no_password, should fail -SYNTAX_ERROR -Replacing existing authentication methods in favor of no_password, should succeed -CREATE USER u01_03174 IDENTIFIED WITH no_password -Trying to auth with no pwd, should succeed -1 -Create user with mix both implicit and explicit auth type, starting with with -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password -Create user with mix both implicit and explicit auth type, starting with by -CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password -Use WITH without providing authentication type, should fail -Syntax error -Create user with ADD identification, should fail, add is not allowed for create query -SYNTAX_ERROR -Trailing comma should result in syntax error -SYNTAX_ERROR -First auth method can't specify type if WITH keyword is not present -SYNTAX_ERROR -RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement -SYNTAX_ERROR -ADD NOT IDENTIFIED should result in syntax error -SYNTAX_ERROR -RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses -SYNTAX_ERROR -On cluster tests localhost 9000 0 0 0 localhost 9000 0 0 0 Basic authentication after user creation diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index 715b99a4334..d0563885892 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -157,8 +157,4 @@ function test } -test "" - -echo "On cluster tests" - test "ON CLUSTER test_shard_localhost" From 7000f214abbb1937423e8282f08bba8431249901 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 29 Aug 2024 09:47:18 -0300 Subject: [PATCH 114/148] add head -n 1 to make sure only on occurrence of error code is grepped --- .../03174_multiple_authentication_methods.sh | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index d0563885892..cd6cb0807f3 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -26,7 +26,7 @@ function test_login_pwd function test_login_pwd_expect_error { - test_login_pwd "$1" "$2" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' + test_login_pwd "$1" "$2" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' | head -n 1 } function test @@ -68,7 +68,7 @@ function test ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH bcrypt_password BY '5'" echo "Asserting SSH does not work anymore" - ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' + ${CLICKHOUSE_CLIENT} --user ${user} --ssh-key-file 'ssh_key' --ssh-key-passphrase "" --query "SELECT 1" 2>&1 | grep -m1 -o 'AUTHENTICATION_FAILED' | head -n 1 echo "Asserting bcrypt_password works" test_login_pwd ${user} '5' @@ -90,12 +90,12 @@ function test test_login_pwd ${user} '6' echo "Multiple identified with, not allowed" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER ${user} $1" echo "CREATE Multiple identified with, not allowed" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '7', IDENTIFIED WITH plaintext_password by '8'" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" @@ -103,13 +103,13 @@ function test ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1" echo "Add identified with, should not be allowed because user is currently identified with no_password and it can not co-exist with other auth types" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '7'" 2>&1 | grep -m1 -o "BAD_ARGUMENTS" | head -n 1 echo "Try to add no_password mixed with other authentication methods, should not be allowed" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '8', no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "Adding no_password, should fail" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH no_password" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "Replacing existing authentication methods in favor of no_password, should succeed" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH no_password" @@ -133,25 +133,25 @@ function test ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" echo "Use WITH without providing authentication type, should fail" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 echo "Create user with ADD identification, should fail, add is not allowed for create query" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "Trailing comma should result in syntax error" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD IDENTIFIED WITH plaintext_password by '1'," 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "First auth method can't specify type if WITH keyword is not present" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED plaintext_password by '1'" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "RESET AUTHENTICATION METHODS TO NEW can only be used on alter statement" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "ADD NOT IDENTIFIED should result in syntax error" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 ADD NOT IDENTIFIED" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 echo "RESET AUTHENTICATION METHODS TO NEW cannot be used along with [ADD] IDENTIFIED clauses" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH plaintext_password by '1' RESET AUTHENTICATION METHODS TO NEW" 2>&1 | grep -m1 -o "SYNTAX_ERROR" | head -n 1 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user}" From edf4e09fb2bc1d8a0a058941f6cb7f13d063c62b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Thu, 29 Aug 2024 18:46:06 +0300 Subject: [PATCH 115/148] Remove stale moving parts without zookeeper --- src/Common/FailPoint.cpp | 1 + .../MergeTree/MergeTreePartsMover.cpp | 16 ++- .../__init__.py | 0 .../test_remove_stale_moving_parts/config.xml | 46 +++++++ .../test_remove_stale_moving_parts/test.py | 126 ++++++++++++++++++ 5 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_remove_stale_moving_parts/__init__.py create mode 100644 tests/integration/test_remove_stale_moving_parts/config.xml create mode 100644 tests/integration/test_remove_stale_moving_parts/test.py diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index b2fcbc77c56..bc4f2cb43d2 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -63,6 +63,7 @@ static struct InitFiu REGULAR(keepermap_fail_drop_data) \ REGULAR(lazy_pipe_fds_fail_close) \ PAUSEABLE(infinite_sleep) \ + PAUSEABLE(stop_moving_part_before_swap_with_active) \ namespace FailPoints diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 9223d6fd5b1..8daa48a2339 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,11 @@ namespace ErrorCodes extern const int DIRECTORY_ALREADY_EXISTS; } +namespace FailPoints +{ + extern const char stop_moving_part_before_swap_with_active[]; +} + namespace { @@ -272,7 +278,13 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part.part = std::move(builder).withPartFormatFromDisk().build(); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath()); - cloned_part.part->is_temp = data->allowRemoveStaleMovingParts(); + if (data->allowRemoveStaleMovingParts()) + { + cloned_part.part->is_temp = data->allowRemoveStaleMovingParts(); + /// Setting it in case connection to zookeeper is lost while moving + /// Otherwise part might be stuck in the moving directory due to the KEEPER_EXCEPTION in part's destructor + cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS; + } cloned_part.part->loadColumnsChecksumsIndexes(true, true); cloned_part.part->loadVersionMetadata(); cloned_part.part->modification_time = cloned_part.part->getDataPartStorage().getLastModified().epochTime(); @@ -282,6 +294,8 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me void MergeTreePartsMover::swapClonedPart(TemporaryClonedPart & cloned_part) const { + /// Used to get some stuck parts in the moving directory by stopping moves while pause is active + FailPointInjection::pauseFailPoint(FailPoints::stop_moving_part_before_swap_with_active); if (moves_blocker.isCancelled()) throw Exception(ErrorCodes::ABORTED, "Cancelled moving parts."); diff --git a/tests/integration/test_remove_stale_moving_parts/__init__.py b/tests/integration/test_remove_stale_moving_parts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_remove_stale_moving_parts/config.xml b/tests/integration/test_remove_stale_moving_parts/config.xml new file mode 100644 index 00000000000..968e07fae51 --- /dev/null +++ b/tests/integration/test_remove_stale_moving_parts/config.xml @@ -0,0 +1,46 @@ + + + + + + ch1 + 9000 + + + + + + 01 + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + + + default + False + + + s3 + False + + + 0.0 + + + + + + true + s3 + + true + diff --git a/tests/integration/test_remove_stale_moving_parts/test.py b/tests/integration/test_remove_stale_moving_parts/test.py new file mode 100644 index 00000000000..dcbcf38d25a --- /dev/null +++ b/tests/integration/test_remove_stale_moving_parts/test.py @@ -0,0 +1,126 @@ +from pathlib import Path +import time +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "config.xml", + ], + macros={"replica": "node1"}, + with_zookeeper=True, + with_minio=True, +) + +DATABASE_NAME = "stale_moving_parts" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(node, query): + return node.query(database=DATABASE_NAME, sql=query) + + +# .../disks/s3/store/ +def get_table_path(node, table): + return ( + node.query( + sql=f"SELECT data_paths FROM system.tables WHERE table = '{table}' and database = '{DATABASE_NAME}' LIMIT 1" + ) + .strip('"\n[]') + .split(",")[1] + .strip("'") + ) + + +def exec(node, cmd, path): + return node.exec_in_container( + [ + "bash", + "-c", + f"{cmd} {path}", + ] + ) + + +def stop_zookeeper(node): + node.exec_in_container(["bash", "-c", "/opt/zookeeper/bin/zkServer.sh stop"]) + timeout = time.time() + 60 + while node.get_process_pid("zookeeper") != None: + if time.time() > timeout: + raise Exception("Failed to stop ZooKeeper in 60 secs") + time.sleep(0.2) + + +def wait_part_is_stuck(node, table_moving_path, moving_part): + num_tries = 5 + while q(node, "SELECT part_name FROM system.moves").strip() != moving_part: + if num_tries == 0: + raise Exception("Part has not started to move") + num_tries -= 1 + time.sleep(1) + num_tries = 5 + while exec(node, "ls", table_moving_path).strip() != moving_part: + if num_tries == 0: + raise Exception("Part is not stuck in the moving directory") + num_tries -= 1 + time.sleep(1) + + +def wait_zookeeper_node_to_start(zk_nodes, timeout=60): + start = time.time() + while time.time() - start < timeout: + try: + for instance in zk_nodes: + conn = cluster.get_kazoo_client(instance) + conn.get_children("/") + print("All instances of ZooKeeper started") + return + except Exception as ex: + print(("Can't connect to ZooKeeper " + str(ex))) + time.sleep(0.5) + + +def test_remove_stale_moving_parts_without_zookeeper(started_cluster): + ch1.query(f"CREATE DATABASE IF NOT EXISTS {DATABASE_NAME}") + + q( + ch1, + "CREATE TABLE test_remove ON CLUSTER cluster ( id UInt32 ) ENGINE ReplicatedMergeTree() ORDER BY id;", + ) + + table_moving_path = Path(get_table_path(ch1, "test_remove")) / "moving" + + q(ch1, "SYSTEM ENABLE FAILPOINT stop_moving_part_before_swap_with_active") + q(ch1, "INSERT INTO test_remove SELECT number FROM numbers(100);") + moving_part = "all_0_0_0" + move_response = ch1.get_query_request( + sql=f"ALTER TABLE test_remove MOVE PART '{moving_part}' TO DISK 's3'", + database=DATABASE_NAME, + ) + + wait_part_is_stuck(ch1, table_moving_path, moving_part) + + cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) + # Stop moves in case table is not read-only yet + q(ch1, "SYSTEM STOP MOVES") + q(ch1, "SYSTEM DISABLE FAILPOINT stop_moving_part_before_swap_with_active") + + assert "Cancelled moving parts" in move_response.get_error() + assert exec(ch1, "ls", table_moving_path).strip() == "" + + cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) + wait_zookeeper_node_to_start(["zoo1", "zoo2", "zoo3"]) + q(ch1, "SYSTEM START MOVES") + + q(ch1, f"DROP TABLE test_remove") From b2c4b771d8f1fc83d4096128231130dc93e9fd79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Thu, 29 Aug 2024 19:33:04 +0300 Subject: [PATCH 116/148] Minor fixes --- src/Storages/MergeTree/MergeTreePartsMover.cpp | 3 ++- tests/integration/test_remove_stale_moving_parts/test.py | 9 --------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 8daa48a2339..11a55fe2420 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -278,9 +278,10 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part.part = std::move(builder).withPartFormatFromDisk().build(); LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part.part->getDataPartStorage().getFullPath()); + cloned_part.part->is_temp = false; if (data->allowRemoveStaleMovingParts()) { - cloned_part.part->is_temp = data->allowRemoveStaleMovingParts(); + cloned_part.part->is_temp = true; /// Setting it in case connection to zookeeper is lost while moving /// Otherwise part might be stuck in the moving directory due to the KEEPER_EXCEPTION in part's destructor cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS; diff --git a/tests/integration/test_remove_stale_moving_parts/test.py b/tests/integration/test_remove_stale_moving_parts/test.py index dcbcf38d25a..f7cb4e5817e 100644 --- a/tests/integration/test_remove_stale_moving_parts/test.py +++ b/tests/integration/test_remove_stale_moving_parts/test.py @@ -53,15 +53,6 @@ def exec(node, cmd, path): ) -def stop_zookeeper(node): - node.exec_in_container(["bash", "-c", "/opt/zookeeper/bin/zkServer.sh stop"]) - timeout = time.time() + 60 - while node.get_process_pid("zookeeper") != None: - if time.time() > timeout: - raise Exception("Failed to stop ZooKeeper in 60 secs") - time.sleep(0.2) - - def wait_part_is_stuck(node, table_moving_path, moving_part): num_tries = 5 while q(node, "SELECT part_name FROM system.moves").strip() != moving_part: From 7c766e7458f3c75c72483b32d5a1adebb7653d93 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 30 Aug 2024 11:41:44 -0300 Subject: [PATCH 117/148] split tests again and randomize user name to be able to parallel --- ...74_multiple_authentication_methods.reference | 9 --------- .../03174_multiple_authentication_methods.sh | 17 ++--------------- ...authentication_methods_show_create.reference | 2 ++ ...tiple_authentication_methods_show_create.sql | 13 +++++++++++++ 4 files changed, 17 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference create mode 100644 tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference index d503b78a2bf..297ef667995 100644 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.reference +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.reference @@ -47,18 +47,9 @@ Adding no_password, should fail SYNTAX_ERROR Replacing existing authentication methods in favor of no_password, should succeed localhost 9000 0 0 0 -CREATE USER u01_03174 IDENTIFIED WITH no_password Trying to auth with no pwd, should succeed 1 localhost 9000 0 0 0 -Create user with mix both implicit and explicit auth type, starting with with -localhost 9000 0 0 0 -CREATE USER u01_03174 IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password -localhost 9000 0 0 0 -Create user with mix both implicit and explicit auth type, starting with by -localhost 9000 0 0 0 -CREATE USER u01_03174 IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password -localhost 9000 0 0 0 Use WITH without providing authentication type, should fail Syntax error Create user with ADD identification, should fail, add is not allowed for create query diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh index cd6cb0807f3..552e5cc20eb 100755 --- a/tests/queries/0_stateless/03174_multiple_authentication_methods.sh +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-parallel, no-replicated-database +# Tags: no-fasttest, no-replicated-database # Tag no-replicated-database: https://s3.amazonaws.com/clickhouse-test-reports/65277/43e9a7ba4bbf7f20145531b384a31304895b55bc/stateless_tests__release__old_analyzer__s3__databasereplicated__[1_2].html and https://github.com/ClickHouse/ClickHouse/blob/011c694117845500c82f9563c65930429979982f/tests/queries/0_stateless/01175_distributed_ddl_output_mode_long.sh#L4 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -31,7 +31,7 @@ function test_login_pwd_expect_error function test { - user="u01_03174" + user="u01_03174$RANDOM" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" @@ -113,25 +113,12 @@ function test echo "Replacing existing authentication methods in favor of no_password, should succeed" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "ALTER USER ${user} $1 IDENTIFIED WITH no_password" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" echo "Trying to auth with no pwd, should succeed" test_login_no_pwd ${user} ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" - echo "Create user with mix both implicit and explicit auth type, starting with with" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4';" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" - - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" - - echo "Create user with mix both implicit and explicit auth type, starting with by" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4';" - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SHOW CREATE USER ${user}" - - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "DROP USER IF EXISTS ${user} $1" - echo "Use WITH without providing authentication type, should fail" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "CREATE USER ${user} $1 IDENTIFIED WITH BY '1';" 2>&1 | grep -m1 -o "Syntax error" | head -n 1 diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference new file mode 100644 index 00000000000..f5da15ccfaa --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.reference @@ -0,0 +1,2 @@ +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED WITH plaintext_password, sha256_password, bcrypt_password, sha256_password +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED WITH sha256_password, plaintext_password, bcrypt_password, sha256_password diff --git a/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql new file mode 100644 index 00000000000..f9e9e72e5c0 --- /dev/null +++ b/tests/queries/0_stateless/03174_multiple_authentication_methods_show_create.sql @@ -0,0 +1,13 @@ +-- Tags: no-fasttest, no-parallel + +-- Create user with mix both implicit and explicit auth type, starting with with +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED WITH plaintext_password by '1', by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_multiple_auth_show_create; + +DROP USER IF EXISTS u_03174_multiple_auth_show_create; + +-- Create user with mix both implicit and explicit auth type, starting with by +CREATE USER u_03174_multiple_auth_show_create IDENTIFIED by '1', plaintext_password by '2', bcrypt_password by '3', by '4'; +SHOW CREATE USER u_03174_multiple_auth_show_create; + +DROP USER IF EXISTS u_03174_multiple_auth_show_create; From 15a67f10dc90c50e9c01377d87c8310ba360c6df Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 2 Sep 2024 09:59:28 -0300 Subject: [PATCH 118/148] add cleanup code to existing tests so flaky tests pass.. --- .../test_enabling_access_management/test.py | 4 ++++ .../test_with_table_engine_grant.py | 2 +- .../integration/test_ssl_cert_authentication/test.py | 8 ++++++++ tests/integration/test_tlsv1_3/test.py | 4 ++++ tests/integration/test_user_valid_until/test.py | 12 ++++++++++++ 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_enabling_access_management/test.py b/tests/integration/test_enabling_access_management/test.py index 821e598d385..1cf05ff9df4 100644 --- a/tests/integration/test_enabling_access_management/test.py +++ b/tests/integration/test_enabling_access_management/test.py @@ -18,6 +18,8 @@ def started_cluster(): def test_enabling_access_management(): + instance.query("DROP USER IF EXISTS Alex") + instance.query("CREATE USER Alex", user="default") assert ( instance.query("SHOW CREATE USER Alex", user="default") @@ -37,3 +39,5 @@ def test_enabling_access_management(): assert "Not enough privileges" in instance.query_and_get_error( "CREATE USER Robin", user="xyz" ) + + instance.query("DROP USER IF EXISTS Alex") diff --git a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py index e181843f1e5..e0139e2547c 100644 --- a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py +++ b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py @@ -36,8 +36,8 @@ def cleanup_after_test(): yield finally: instance.query("DROP USER IF EXISTS A, B, C") - instance.query("DROP TABLE IF EXISTS test.view_1") + instance.query("DROP TABLE IF EXISTS test.view_1, test.view_2, default.table") def test_smoke(): instance.query("CREATE USER A") diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 603ff8f3116..94d8df2fcef 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -297,6 +297,8 @@ def test_https_non_ssl_auth(): def test_create_user(): + instance.query("DROP USER IF EXISTS emma") + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") assert ( execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") @@ -334,8 +336,12 @@ def test_create_user(): 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) + instance.query("DROP USER IF EXISTS emma") + def test_x509_san_support(): + instance.query("DROP USER IF EXISTS jemma") + assert ( execute_query_native( instance, "SELECT currentUser()", user="jerome", cert_name="client4" @@ -369,3 +375,5 @@ def test_x509_san_support(): instance.query("SHOW CREATE USER jemma") == "CREATE USER jemma IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://foo.com/bar\\', \\'URI:spiffe://foo.com/baz\\'\n" ) + + instance.query("DROP USER IF EXISTS jemma") diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py index 07239c1b31f..f88ff5da814 100644 --- a/tests/integration/test_tlsv1_3/test.py +++ b/tests/integration/test_tlsv1_3/test.py @@ -186,6 +186,8 @@ def test_https_non_ssl_auth(): def test_create_user(): + instance.query("DROP USER IF EXISTS emma") + instance.query("CREATE USER emma IDENTIFIED WITH ssl_certificate CN 'client3'") assert ( execute_query_https("SELECT currentUser()", user="emma", cert_name="client3") @@ -222,3 +224,5 @@ def test_create_user(): == "emma\t['ssl_certificate']\t['{\"common_names\":[\"client2\"]}']\n" 'lucy\t[\'ssl_certificate\']\t[\'{"common_names":["client2","client3"]}\']\n' ) + + instance.query("DROP USER IF EXISTS emma") diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py index f25b0021913..50b7dd098c9 100644 --- a/tests/integration/test_user_valid_until/test.py +++ b/tests/integration/test_user_valid_until/test.py @@ -19,6 +19,8 @@ def started_cluster(): def test_basic(started_cluster): + node.query("DROP USER IF EXISTS user_basic") + # 1. Without VALID UNTIL node.query("CREATE USER user_basic") @@ -71,8 +73,12 @@ def test_basic(started_cluster): error = "Authentication failed" assert error in node.query_and_get_error("SELECT 1", user="user_basic") + node.query("DROP USER IF EXISTS user_basic") + def test_details(started_cluster): + node.query("DROP USER IF EXISTS user_details_infinity, user_details_time_only") + # 1. Does not do anything node.query("CREATE USER user_details_infinity VALID UNTIL 'infinity'") @@ -93,8 +99,12 @@ def test_details(started_cluster): == f"CREATE USER user_details_time_only IDENTIFIED WITH no_password VALID UNTIL \\'{until_year}-01-01 22:03:40\\'\n" ) + node.query("DROP USER IF EXISTS user_details_infinity, user_details_time_only") + def test_restart(started_cluster): + node.query("DROP USER IF EXISTS user_restart") + node.query("CREATE USER user_restart VALID UNTIL '06/11/2010 08:03:20 Z+3'") assert ( @@ -111,3 +121,5 @@ def test_restart(started_cluster): error = "Authentication failed" assert error in node.query_and_get_error("SELECT 1", user="user_restart") + + node.query("DROP USER IF EXISTS user_restart") From 0bf95655aa79b4dca642a08d0fab2666fb2e0bf7 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Mon, 2 Sep 2024 10:45:37 -0300 Subject: [PATCH 119/148] add new line --- .../test_grant_and_revoke/test_with_table_engine_grant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py index e0139e2547c..3c579b68177 100644 --- a/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py +++ b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py @@ -39,6 +39,7 @@ def cleanup_after_test(): instance.query("DROP TABLE IF EXISTS test.view_1, test.view_2, default.table") + def test_smoke(): instance.query("CREATE USER A") assert "Not enough privileges" in instance.query_and_get_error( From cf12e3924f51fd08c884519352a46495412a771a Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 11 Sep 2024 09:31:46 +0000 Subject: [PATCH 120/148] Speedup insert data with vector similarity index by add data to index parallel --- src/Common/CurrentMetrics.cpp | 3 ++ .../MergeTreeIndexVectorSimilarity.cpp | 42 +++++++++++++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 4bf2b0704f1..1d37b4761e1 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -178,6 +178,9 @@ M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \ M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \ M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \ + M(UsearchUpdateThreads, "Number of threads in the Aggregator thread pool.") \ + M(UsearchUpdateThreadsActive, "Number of threads in the Aggregator thread pool running a task.") \ + M(UsearchUpdateThreadsScheduled, "Number of queued or active jobs in the Aggregator thread pool.") \ \ M(DiskPlainRewritableAzureDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \ M(DiskPlainRewritableLocalDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for LocalObjectStorage.") \ diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 58892d0dbf2..ad166839ce3 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -4,16 +4,20 @@ #include #include +#include #include +#include #include #include #include +#include #include #include #include #include #include + namespace ProfileEvents { extern const Event USearchAddCount; @@ -24,6 +28,13 @@ namespace ProfileEvents extern const Event USearchSearchComputedDistances; } +namespace CurrentMetrics +{ +extern const Metric UsearchUpdateThreads; +extern const Metric UsearchUpdateThreadsActive; +extern const Metric UsearchUpdateThreadsScheduled; +} + namespace DB { @@ -273,17 +284,42 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + rows))) throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index"); - for (size_t row = 0; row < rows; ++row) + size_t max_threads = Context::getGlobalContextInstance()->getSettingsRef().max_threads; + max_threads = max_threads > 0 ? max_threads : getNumberOfPhysicalCPUCores(); + + auto thread_pool = std::make_unique( + CurrentMetrics::UsearchUpdateThreads, + CurrentMetrics::UsearchUpdateThreadsActive, + CurrentMetrics::UsearchUpdateThreadsScheduled, + max_threads); + + auto add_vector_to_index = [&](USearchIndex::vector_key_t key, size_t row, ThreadGroupPtr thread_group) { - if (auto result = index->add(static_cast(index->size()), &column_array_data_float_data[column_array_offsets[row - 1]]); !result) - throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release())); + SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached();); + + if (thread_group) + CurrentThread::attachToGroupIfDetached(thread_group); + + if (auto result = index->add(key, &column_array_data_float_data[column_array_offsets[row - 1]]); !result) + throw Exception( + ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release())); else { ProfileEvents::increment(ProfileEvents::USearchAddCount); ProfileEvents::increment(ProfileEvents::USearchAddVisitedMembers, result.visited_members); ProfileEvents::increment(ProfileEvents::USearchAddComputedDistances, result.computed_distances); } + }; + + size_t current_index_size = index->size(); + + for (size_t row = 0; row < rows; ++row) + { + auto key = static_cast(current_index_size + row); + auto task = [group = CurrentThread::getGroup(), &add_vector_to_index, key, row] { add_vector_to_index(key, row, group); }; + thread_pool->scheduleOrThrowOnError(task); } + thread_pool->wait(); } } From 7425d4aa1ae690810db444796ec4c0a4469d3e76 Mon Sep 17 00:00:00 2001 From: flynn Date: Wed, 11 Sep 2024 10:12:42 +0000 Subject: [PATCH 121/148] remove blank line --- src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index ad166839ce3..6379c837281 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -17,7 +17,6 @@ #include #include - namespace ProfileEvents { extern const Event USearchAddCount; From 22c3b71196bc15c3229edae5f6abd7e59950bbe6 Mon Sep 17 00:00:00 2001 From: flynn Date: Thu, 12 Sep 2024 03:54:25 +0000 Subject: [PATCH 122/148] Make vector similarity index creation thread pool globally --- src/Core/ServerSettings.h | 3 ++- src/Interpreters/Context.cpp | 26 ++++++++++++++++++- src/Interpreters/Context.h | 2 ++ .../MergeTreeIndexVectorSimilarity.cpp | 21 +++------------ 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 79173503f28..2077967e5b6 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -50,7 +50,7 @@ namespace DB M(UInt32, asynchronous_heavy_metrics_update_period_s, 120, "Period in seconds for updating heavy asynchronous metrics.", 0) \ M(String, default_database, "default", "Default database name.", 0) \ M(String, tmp_policy, "", "Policy for storage with temporary data.", 0) \ - M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting., ", 0) \ + M(UInt64, max_temporary_data_on_disk_size, 0, "The maximum amount of storage that could be used for external aggregation, joins or sorting.", 0) \ M(String, temporary_data_in_cache, "", "Cache disk name for temporary data.", 0) \ M(UInt64, aggregate_function_group_array_max_element_size, 0xFFFFFF, "Max array element size in bytes for groupArray function. This limit is checked at serialization and help to avoid large state size.", 0) \ M(GroupArrayActionWhenLimitReached, aggregate_function_group_array_action_when_limit_is_reached, GroupArrayActionWhenLimitReached::THROW, "Action to execute when max array element size is exceeded in groupArray: `throw` exception, or `discard` extra values", 0) \ @@ -65,6 +65,7 @@ namespace DB M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \ M(Bool, async_insert_queue_flush_on_shutdown, true, "If true queue of asynchronous inserts is flushed on graceful shutdown", 0) \ M(Bool, ignore_empty_sql_security_in_create_view_query, true, "If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. This setting is only necessary for the migration period and will become obsolete in 24.4", 0) \ + M(UInt64, max_thread_pool_size_for_vector_similarity_index_creation, 16, "The maximum number of threads that could be allocated from the OS and used for vector similarity index creation(0 means all cores).", 0) \ \ /* Database Catalog */ \ M(UInt64, database_atomic_delay_before_drop_table_sec, 8 * 60, "The delay during which a dropped table can be restored using the UNDROP statement. If DROP TABLE ran with a SYNC modifier, the setting is ignored.", 0) \ diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 373cc91ebcb..051378ef6d4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -120,7 +121,6 @@ #include #include - namespace fs = std::filesystem; namespace ProfileEvents @@ -168,6 +168,9 @@ namespace CurrentMetrics extern const Metric AttachedDictionary; extern const Metric AttachedDatabase; extern const Metric PartsActive; + extern const Metric UsearchUpdateThreads; + extern const Metric UsearchUpdateThreadsActive; + extern const Metric UsearchUpdateThreadsScheduled; } @@ -296,6 +299,8 @@ struct ContextSharedPart : boost::noncopyable mutable std::unique_ptr load_marks_threadpool; /// Threadpool for loading marks cache. mutable OnceFlag prefetch_threadpool_initialized; mutable std::unique_ptr prefetch_threadpool; /// Threadpool for loading marks cache. + mutable OnceFlag vector_similarity_index_creation_threadpool_initialized; + mutable std::unique_ptr vector_similarity_index_creation_threadpool; /// Threadpool for vector-similarity index creation. mutable UncompressedCachePtr index_uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks for MergeTree indices. mutable QueryCachePtr query_cache TSA_GUARDED_BY(mutex); /// Cache of query results. mutable MarkCachePtr index_mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files of MergeTree indices. @@ -3095,6 +3100,25 @@ ThreadPool & Context::getLoadMarksThreadpool() const return *shared->load_marks_threadpool; } +ThreadPool & Context::getVectorSimilarityIndexCreationThreadPool() const +{ + callOnce( + shared->vector_similarity_index_creation_threadpool_initialized, + [&] + { + const auto & server_setting = getServerSettings(); + size_t max_thread_pool_size = server_setting.max_thread_pool_size_for_vector_similarity_index_creation > 0 + ? server_setting.max_thread_pool_size_for_vector_similarity_index_creation + : getNumberOfPhysicalCPUCores(); + shared->vector_similarity_index_creation_threadpool = std::make_unique( + CurrentMetrics::UsearchUpdateThreads, + CurrentMetrics::UsearchUpdateThreadsActive, + CurrentMetrics::UsearchUpdateThreadsScheduled, + max_thread_pool_size); + }); + return *shared->vector_similarity_index_creation_threadpool; +} + void Context::setIndexUncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index fb5337158ba..e8f6ce9b3e1 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1096,6 +1096,8 @@ public: /// and make a prefetch by putting a read task to threadpoolReader. size_t getPrefetchThreadpoolSize() const; + ThreadPool & getVectorSimilarityIndexCreationThreadPool() const; + /// Settings for MergeTree background tasks stored in config.xml BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const; BackgroundTaskSchedulingSettings getBackgroundMoveTaskSchedulingSettings() const; diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 6379c837281..e5ed19585f8 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -27,13 +26,6 @@ namespace ProfileEvents extern const Event USearchSearchComputedDistances; } -namespace CurrentMetrics -{ -extern const Metric UsearchUpdateThreads; -extern const Metric UsearchUpdateThreadsActive; -extern const Metric UsearchUpdateThreadsScheduled; -} - namespace DB { @@ -283,14 +275,7 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + rows))) throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index"); - size_t max_threads = Context::getGlobalContextInstance()->getSettingsRef().max_threads; - max_threads = max_threads > 0 ? max_threads : getNumberOfPhysicalCPUCores(); - - auto thread_pool = std::make_unique( - CurrentMetrics::UsearchUpdateThreads, - CurrentMetrics::UsearchUpdateThreadsActive, - CurrentMetrics::UsearchUpdateThreadsScheduled, - max_threads); + auto & thread_pool = Context::getGlobalContextInstance()->getVectorSimilarityIndexCreationThreadPool(); auto add_vector_to_index = [&](USearchIndex::vector_key_t key, size_t row, ThreadGroupPtr thread_group) { @@ -316,9 +301,9 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c { auto key = static_cast(current_index_size + row); auto task = [group = CurrentThread::getGroup(), &add_vector_to_index, key, row] { add_vector_to_index(key, row, group); }; - thread_pool->scheduleOrThrowOnError(task); + thread_pool.scheduleOrThrowOnError(task); } - thread_pool->wait(); + thread_pool.wait(); } } From fe5e061fffe3ef448dfc8d22b5f1236b09e036ca Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 12 Sep 2024 10:16:29 +0000 Subject: [PATCH 123/148] Some fixups --- .../mergetree-family/annindexes.md | 4 ++ .../settings.md | 8 ++++ src/Common/CurrentMetrics.cpp | 6 +-- src/Core/ServerSettings.h | 2 +- src/Interpreters/Context.cpp | 44 +++++++++---------- src/Interpreters/Context.h | 2 +- .../MergeTreeIndexVectorSimilarity.cpp | 29 ++++++------ 7 files changed, 53 insertions(+), 42 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index 3c75b8dbef0..b73700c40f4 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -107,6 +107,10 @@ The vector similarity index currently does not work with per-table, non-default [here](https://github.com/ClickHouse/ClickHouse/pull/51325#issuecomment-1605920475)). If necessary, the value must be changed in config.xml. ::: +Vector index creation is known to be slow. To speed the process up, index creation can be parallelized. The maximum number of threads can be +configured using server configuration +setting [max_build_vector_similarity_index_thread_pool_size](server-configuration-parameters/settings.md#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size). + ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write requests. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index ccc8cf017ca..14a23964100 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -491,6 +491,14 @@ Type: Double Default: 0.9 +## max_build_vector_similarity_index_thread_pool_size {#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size} + +The maximum number of threads to use for building vector indexes. 0 means all cores. + +Type: UInt64 + +Default: 16 + ## cgroups_memory_usage_observer_wait_time Interval in seconds during which the server's maximum allowed memory consumption is adjusted by the corresponding threshold in cgroups. (see diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 1d37b4761e1..658eaedbda1 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -178,9 +178,9 @@ M(ObjectStorageAzureThreads, "Number of threads in the AzureObjectStorage thread pool.") \ M(ObjectStorageAzureThreadsActive, "Number of threads in the AzureObjectStorage thread pool running a task.") \ M(ObjectStorageAzureThreadsScheduled, "Number of queued or active jobs in the AzureObjectStorage thread pool.") \ - M(UsearchUpdateThreads, "Number of threads in the Aggregator thread pool.") \ - M(UsearchUpdateThreadsActive, "Number of threads in the Aggregator thread pool running a task.") \ - M(UsearchUpdateThreadsScheduled, "Number of queued or active jobs in the Aggregator thread pool.") \ + M(BuildVectorSimilarityIndexThreads, "Number of threads in the build vector similarity index thread pool.") \ + M(BuildVectorSimilarityIndexThreadsActive, "Number of threads in the build vector similarity index thread pool running a task.") \ + M(BuildVectorSimilarityIndexThreadsScheduled, "Number of queued or active jobs in the build vector similarity index thread pool.") \ \ M(DiskPlainRewritableAzureDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for AzureObjectStorage.") \ M(DiskPlainRewritableLocalDirectoryMapSize, "Number of local-to-remote path entries in the 'plain_rewritable' in-memory map for LocalObjectStorage.") \ diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index b5071e6fa5d..689b18cb74f 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -65,7 +65,7 @@ namespace DB M(UInt64, async_insert_threads, 16, "Maximum number of threads to actually parse and insert data in background. Zero means asynchronous mode is disabled", 0) \ M(Bool, async_insert_queue_flush_on_shutdown, true, "If true queue of asynchronous inserts is flushed on graceful shutdown", 0) \ M(Bool, ignore_empty_sql_security_in_create_view_query, true, "If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. This setting is only necessary for the migration period and will become obsolete in 24.4", 0) \ - M(UInt64, max_thread_pool_size_for_vector_similarity_index_creation, 16, "The maximum number of threads that could be allocated from the OS and used for vector similarity index creation(0 means all cores).", 0) \ + M(UInt64, max_build_vector_similarity_index_thread_pool_size, 16, "The maximum number of threads to use to build vector similarity indexes. 0 means all cores.", 0) \ \ /* Database Catalog */ \ M(UInt64, database_atomic_delay_before_drop_table_sec, 8 * 60, "The delay during which a dropped table can be restored using the UNDROP statement. If DROP TABLE ran with a SYNC modifier, the setting is ignored.", 0) \ diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7350fdecd25..311fd094706 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -164,14 +164,14 @@ namespace CurrentMetrics extern const Metric TablesLoaderForegroundThreadsActive; extern const Metric TablesLoaderForegroundThreadsScheduled; extern const Metric IOWriterThreadsScheduled; + extern const Metric BuildVectorSimilarityIndexThreads; + extern const Metric BuildVectorSimilarityIndexThreadsActive; + extern const Metric BuildVectorSimilarityIndexThreadsScheduled; extern const Metric AttachedTable; extern const Metric AttachedView; extern const Metric AttachedDictionary; extern const Metric AttachedDatabase; extern const Metric PartsActive; - extern const Metric UsearchUpdateThreads; - extern const Metric UsearchUpdateThreadsActive; - extern const Metric UsearchUpdateThreadsScheduled; } @@ -300,8 +300,8 @@ struct ContextSharedPart : boost::noncopyable mutable std::unique_ptr load_marks_threadpool; /// Threadpool for loading marks cache. mutable OnceFlag prefetch_threadpool_initialized; mutable std::unique_ptr prefetch_threadpool; /// Threadpool for loading marks cache. - mutable OnceFlag vector_similarity_index_creation_threadpool_initialized; - mutable std::unique_ptr vector_similarity_index_creation_threadpool; /// Threadpool for vector-similarity index creation. + mutable OnceFlag build_vector_similarity_index_threadpool_initialized; + mutable std::unique_ptr build_vector_similarity_index_threadpool; /// Threadpool for vector-similarity index creation. mutable UncompressedCachePtr index_uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks for MergeTree indices. mutable QueryCachePtr query_cache TSA_GUARDED_BY(mutex); /// Cache of query results. mutable MarkCachePtr index_mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files of MergeTree indices. @@ -3101,25 +3101,6 @@ ThreadPool & Context::getLoadMarksThreadpool() const return *shared->load_marks_threadpool; } -ThreadPool & Context::getVectorSimilarityIndexCreationThreadPool() const -{ - callOnce( - shared->vector_similarity_index_creation_threadpool_initialized, - [&] - { - const auto & server_setting = getServerSettings(); - size_t max_thread_pool_size = server_setting.max_thread_pool_size_for_vector_similarity_index_creation > 0 - ? server_setting.max_thread_pool_size_for_vector_similarity_index_creation - : getNumberOfPhysicalCPUCores(); - shared->vector_similarity_index_creation_threadpool = std::make_unique( - CurrentMetrics::UsearchUpdateThreads, - CurrentMetrics::UsearchUpdateThreadsActive, - CurrentMetrics::UsearchUpdateThreadsScheduled, - max_thread_pool_size); - }); - return *shared->vector_similarity_index_creation_threadpool; -} - void Context::setIndexUncompressedCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); @@ -3321,6 +3302,21 @@ size_t Context::getPrefetchThreadpoolSize() const return config.getUInt(".prefetch_threadpool_pool_size", 100); } +ThreadPool & Context::getBuildVectorSimilarityIndexThreadPool() const +{ + callOnce(shared->build_vector_similarity_index_threadpool_initialized, [&] { + size_t pool_size = shared->server_settings.max_build_vector_similarity_index_thread_pool_size > 0 + ? shared->server_settings.max_build_vector_similarity_index_thread_pool_size + : getNumberOfPhysicalCPUCores(); + shared->build_vector_similarity_index_threadpool = std::make_unique( + CurrentMetrics::BuildVectorSimilarityIndexThreads, + CurrentMetrics::BuildVectorSimilarityIndexThreadsActive, + CurrentMetrics::BuildVectorSimilarityIndexThreadsScheduled, + pool_size); + }); + return *shared->build_vector_similarity_index_threadpool; +} + BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const { callOnce(shared->buffer_flush_schedule_pool_initialized, [&] { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index d309b964dbd..0daef2243aa 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1097,7 +1097,7 @@ public: /// and make a prefetch by putting a read task to threadpoolReader. size_t getPrefetchThreadpoolSize() const; - ThreadPool & getVectorSimilarityIndexCreationThreadPool() const; + ThreadPool & getBuildVectorSimilarityIndexThreadPool() const; /// Settings for MergeTree background tasks stored in config.xml BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const; diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index e5ed19585f8..8a850141a67 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -4,12 +4,10 @@ #include #include -#include #include #include #include #include -#include #include #include #include @@ -31,7 +29,6 @@ namespace DB namespace ErrorCodes { - extern const int CANNOT_ALLOCATE_MEMORY; extern const int FORMAT_VERSION_TOO_OLD; extern const int ILLEGAL_COLUMN; extern const int INCORRECT_DATA; @@ -133,8 +130,7 @@ void USearchIndexWithSerialization::deserialize(ReadBuffer & istr) /// See the comment in MergeTreeIndexGranuleVectorSimilarity::deserializeBinary why we throw here throw Exception(ErrorCodes::INCORRECT_DATA, "Could not load vector similarity index. Please drop the index and create it again. Error: {}", String(result.error.release())); - if (!try_reserve(limits())) - throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for usearch index"); + try_reserve(limits()); } USearchIndexWithSerialization::Statistics USearchIndexWithSerialization::getStatistics() const @@ -272,21 +268,27 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length"); /// Reserving space is mandatory - if (!index->try_reserve(roundUpToPowerOfTwoOrZero(index->size() + rows))) - throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not reserve memory for vector similarity index"); + index->reserve(roundUpToPowerOfTwoOrZero(index->size() + rows)); - auto & thread_pool = Context::getGlobalContextInstance()->getVectorSimilarityIndexCreationThreadPool(); + /// Vector index creation is slooooow. Add the new rows in parallel. The threadpool is global to avoid oversubscription when multiple + /// indexes are build simultaneously (e.g. multiple merges run at the same time). + auto & thread_pool = Context::getGlobalContextInstance()->getBuildVectorSimilarityIndexThreadPool(); auto add_vector_to_index = [&](USearchIndex::vector_key_t key, size_t row, ThreadGroupPtr thread_group) { - SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached();); + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachFromGroupIfNotDetached(); + ); if (thread_group) CurrentThread::attachToGroupIfDetached(thread_group); + /// add is thread-safe if (auto result = index->add(key, &column_array_data_float_data[column_array_offsets[row - 1]]); !result) - throw Exception( - ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release())); + { + throw Exception(ErrorCodes::INCORRECT_DATA, "Could not add data to vector similarity index. Error: {}", String(result.error.release())); + } else { ProfileEvents::increment(ProfileEvents::USearchAddCount); @@ -295,14 +297,15 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c } }; - size_t current_index_size = index->size(); + size_t index_size = index->size(); for (size_t row = 0; row < rows; ++row) { - auto key = static_cast(current_index_size + row); + auto key = static_cast(index_size + row); auto task = [group = CurrentThread::getGroup(), &add_vector_to_index, key, row] { add_vector_to_index(key, row, group); }; thread_pool.scheduleOrThrowOnError(task); } + thread_pool.wait(); } From 38b5ea9066a8eae29222e26595957d022d2de26c Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 12 Sep 2024 12:43:27 +0000 Subject: [PATCH 124/148] Fix docs --- docs/en/engines/table-engines/mergetree-family/annindexes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index b73700c40f4..f507e2b9f86 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -109,7 +109,7 @@ The vector similarity index currently does not work with per-table, non-default Vector index creation is known to be slow. To speed the process up, index creation can be parallelized. The maximum number of threads can be configured using server configuration -setting [max_build_vector_similarity_index_thread_pool_size](server-configuration-parameters/settings.md#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size). +setting [max_build_vector_similarity_index_thread_pool_size](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters_max_build_vector_similarity_index_thread_pool_size). ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write From 042194e3f6e2ad940a3cedc921b771808d89305a Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 13 Sep 2024 08:50:28 +0000 Subject: [PATCH 125/148] enable removeRecursive in CI --- tests/config/config.d/keeper_port.xml | 1 + tests/docker_scripts/stress_tests.lib | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index 2b04d843a3b..709d6641806 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -42,6 +42,7 @@ 1 1 1 + 1 diff --git a/tests/docker_scripts/stress_tests.lib b/tests/docker_scripts/stress_tests.lib index cc4c290afef..fb63ef81f80 100644 --- a/tests/docker_scripts/stress_tests.lib +++ b/tests/docker_scripts/stress_tests.lib @@ -64,6 +64,7 @@ function configure() randomize_config_boolean_value multi_read keeper_port randomize_config_boolean_value check_not_exists keeper_port randomize_config_boolean_value create_if_not_exists keeper_port + randomize_config_boolean_value remove_recursive keeper_port fi sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml From 9ca149a487beef10de0864f1381927ebf4514b76 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 13 Sep 2024 11:07:09 +0000 Subject: [PATCH 126/148] Fix GWP-asan crash --- src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 8a850141a67..641770b16e9 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -268,7 +269,9 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c throw Exception(ErrorCodes::INCORRECT_DATA, "All arrays in column with vector similarity index must have equal length"); /// Reserving space is mandatory - index->reserve(roundUpToPowerOfTwoOrZero(index->size() + rows)); + size_t max_thread_pool_size = Context::getGlobalContextInstance()->getServerSettings().max_build_vector_similarity_index_thread_pool_size; + unum::usearch::index_limits_t limits(roundUpToPowerOfTwoOrZero(index->size() + rows), max_thread_pool_size); + index->reserve(limits); /// Vector index creation is slooooow. Add the new rows in parallel. The threadpool is global to avoid oversubscription when multiple /// indexes are build simultaneously (e.g. multiple merges run at the same time). From baf6aaef1dc31bc17209cd1be68a860d7c01cfec Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 13 Sep 2024 11:32:33 +0000 Subject: [PATCH 127/148] fix tests --- .../0_stateless/02735_system_zookeeper_connection.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02735_system_zookeeper_connection.reference b/tests/queries/0_stateless/02735_system_zookeeper_connection.reference index a2e987666e4..4eb6092137b 100644 --- a/tests/queries/0_stateless/02735_system_zookeeper_connection.reference +++ b/tests/queries/0_stateless/02735_system_zookeeper_connection.reference @@ -1,2 +1,2 @@ -default 127.0.0.1 9181 0 0 0 1 1 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS'] +default 127.0.0.1 9181 0 0 0 1 1 ['FILTERED_LIST','MULTI_READ','CHECK_NOT_EXISTS','CREATE_IF_NOT_EXISTS','REMOVE_RECURSIVE'] zookeeper2 localhost 9181 0 0 0 1 From ae6e236acf41ccb97d9e0f9864a74cffbf746db9 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 13 Sep 2024 08:59:59 -0300 Subject: [PATCH 128/148] update recently introduced test --- tests/integration/test_ssl_cert_authentication/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 6c07f91bd51..9695e0fbd5d 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -391,7 +391,7 @@ def test_x509_san_wildcard_support(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name='stewie'" ) - == 'stewie\tssl_certificate\t{"subject_alt_names":["URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far"]}\n' + == 'stewie\t[\'ssl_certificate\']\t[\'{"subject_alt_names":["URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far"]}\']\n' ) assert ( From 54e75fd4dcde66d9c5c077b276530e57172a2fee Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Fri, 13 Sep 2024 09:04:14 -0300 Subject: [PATCH 129/148] fix black --- tests/integration/test_ssl_cert_authentication/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 9695e0fbd5d..19ad67ec136 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -391,7 +391,7 @@ def test_x509_san_wildcard_support(): instance.query( "SELECT name, auth_type, auth_params FROM system.users WHERE name='stewie'" ) - == 'stewie\t[\'ssl_certificate\']\t[\'{"subject_alt_names":["URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far"]}\']\n' + == "stewie\t['ssl_certificate']\t['{\"subject_alt_names\":[\"URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far\"]}']\n" ) assert ( From 6a7cfd13f7c8840617abefc40f1139b234f8ff59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=B0=D1=80?= =?UTF-8?q?=D0=B1=D0=B0=D1=80?= Date: Fri, 13 Sep 2024 15:25:17 +0300 Subject: [PATCH 130/148] Set PRESERVE_BLOBS if part is fetched from another replica --- src/Storages/MergeTree/MergeTreePartsMover.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 11a55fe2420..d81300da738 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -232,6 +232,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part.temporary_directory_lock = data->getTemporaryPartDirectoryHolder(part->name); MutableDataPartStoragePtr cloned_part_storage; + bool preserve_blobs = false; if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { /// Try zero-copy replication and fallback to default copy if it's not possible @@ -259,6 +260,7 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me if (zero_copy_part) { /// FIXME for some reason we cannot just use this part, we have to re-create it through MergeTreeDataPartBuilder + preserve_blobs = true; zero_copy_part->is_temp = false; /// Do not remove it in dtor cloned_part_storage = zero_copy_part->getDataPartStoragePtr(); } @@ -284,7 +286,10 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me cloned_part.part->is_temp = true; /// Setting it in case connection to zookeeper is lost while moving /// Otherwise part might be stuck in the moving directory due to the KEEPER_EXCEPTION in part's destructor - cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS; + if (preserve_blobs) + cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::PRESERVE_BLOBS; + else + cloned_part.part->remove_tmp_policy = IMergeTreeDataPart::BlobsRemovalPolicyForTemporaryParts::REMOVE_BLOBS; } cloned_part.part->loadColumnsChecksumsIndexes(true, true); cloned_part.part->loadVersionMetadata(); From 2b6238a5e548226275fefece93684296fca2b5f5 Mon Sep 17 00:00:00 2001 From: Michael Stetsyuk Date: Fri, 13 Sep 2024 13:41:01 +0000 Subject: [PATCH 131/148] sleep for 10ms beforen retrying to acquire a lock in DatabaseReplicatedDDLWorker::enqueueQueryImpl --- src/Databases/DatabaseReplicatedWorker.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 4e7408aa96e..07150957912 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; @@ -249,6 +250,8 @@ String DatabaseReplicatedDDLWorker::enqueueQueryImpl(const ZooKeeperPtr & zookee } else if (code != Coordination::Error::ZNODEEXISTS) zkutil::KeeperMultiException::check(code, ops, res); + + std::this_thread::sleep_for(std::chrono::milliseconds(50)); } if (counter_path.empty()) From b420bbf855f9126db774035b1672565484d947a2 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 13 Sep 2024 17:17:10 +0200 Subject: [PATCH 132/148] Improve debug action --- .github/actions/debug/action.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/actions/debug/action.yml b/.github/actions/debug/action.yml index e1fe3f28024..d116c422e03 100644 --- a/.github/actions/debug/action.yml +++ b/.github/actions/debug/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | echo "::group::Envs" - env + env | sort echo "::endgroup::" - name: Print Event.json shell: bash @@ -16,3 +16,23 @@ runs: echo "::group::Event.json" python3 -m json.tool "$GITHUB_EVENT_PATH" echo "::endgroup::" + - name: Pring contexts + shell: bash + run: | + cat << 'EOF' + ::group::github + ${{ toJSON(github) }} + ::endgroup:: + + ::group::runner + ${{ toJSON(runner) }} + ::endgroup:: + + ::group::env + ${{ toJSON(env) }} + ::endgroup:: + + ::group::job + ${{ toJSON(job) }} + ::endgroup:: + EOF From 418ef3f8bc9aebbc3f7e644b19b3b6c44f33e8da Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 13 Sep 2024 17:20:49 +0200 Subject: [PATCH 133/148] Use local debug action in every action --- .github/workflows/backport_branches.yml | 2 ++ .github/workflows/cherry_pick.yml | 2 ++ .github/workflows/create_release.yml | 4 ++-- .github/workflows/docker_test_images.yml | 1 + .github/workflows/jepsen.yml | 7 ++++--- .github/workflows/master.yml | 4 ++-- .github/workflows/merge_queue.yml | 4 ++-- .github/workflows/nightly.yml | 4 ++-- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/release_branches.yml | 2 ++ .github/workflows/reusable_simple_job.yml | 4 ++-- 11 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 23744dc7f8f..794aca4a515 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -27,6 +27,8 @@ jobs: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Labels check run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index 8d1e2055978..315673d4abc 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -33,6 +33,8 @@ jobs: clear-repository: true token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} fetch-depth: 0 + - name: Debug Info + uses: ./.github/actions/debug - name: Cherry pick run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 1fb6cb60e96..b6c460ab37c 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -56,13 +56,13 @@ jobs: GH_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }} runs-on: [self-hosted, release-maker] steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} fetch-depth: 0 + - name: Debug Info + uses: ./.github/actions/debug - name: Prepare Release Info shell: bash run: | diff --git a/.github/workflows/docker_test_images.yml b/.github/workflows/docker_test_images.yml index 3fe1a8883c6..2138420f378 100644 --- a/.github/workflows/docker_test_images.yml +++ b/.github/workflows/docker_test_images.yml @@ -11,6 +11,7 @@ name: Build docker images required: false type: boolean default: false + jobs: DockerBuildAarch64: runs-on: [self-hosted, style-checker-aarch64] diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml index ecafde9e4cb..92e4ce10ade 100644 --- a/.github/workflows/jepsen.yml +++ b/.github/workflows/jepsen.yml @@ -8,27 +8,28 @@ on: # yamllint disable-line rule:truthy schedule: - cron: '0 */6 * * *' workflow_dispatch: + jobs: RunConfig: runs-on: [self-hosted, style-checker-aarch64] outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: PrepareRunConfig id: runconfig run: | echo "::group::configure CI run" python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow "$GITHUB_WORKFLOW" --outfile ${{ runner.temp }}/ci_run_data.json echo "::endgroup::" - + echo "::group::CI run configure results" python3 -m json.tool ${{ runner.temp }}/ci_run_data.json echo "::endgroup::" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2ce1124404f..b76bbbbbdbe 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -15,14 +15,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Merge sync PR run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml index 629cf79770e..45ce81c2caf 100644 --- a/.github/workflows/merge_queue.yml +++ b/.github/workflows/merge_queue.yml @@ -14,14 +14,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Cancel PR workflow run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 36fea39686f..1cea94e7500 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,14 +15,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: PrepareRunConfig id: runconfig run: | diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index dbc740ebc1b..acd392978b6 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -25,14 +25,14 @@ jobs: outputs: data: ${{ steps.runconfig.outputs.CI_DATA }} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get a version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Cancel previous Sync PR workflow run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index ec119b6ff95..b884ebfe7a0 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -24,6 +24,8 @@ jobs: clear-repository: true # to ensure correct digests fetch-depth: 0 # to get version filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Labels check run: | cd "$GITHUB_WORKSPACE/tests/ci" diff --git a/.github/workflows/reusable_simple_job.yml b/.github/workflows/reusable_simple_job.yml index 4d48662ae4e..7df98d96f79 100644 --- a/.github/workflows/reusable_simple_job.yml +++ b/.github/workflows/reusable_simple_job.yml @@ -62,8 +62,6 @@ jobs: env: GITHUB_JOB_OVERRIDDEN: ${{inputs.test_name}} steps: - - name: DebugInfo - uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 - name: Check out repository code uses: ClickHouse/checkout@v1 with: @@ -72,6 +70,8 @@ jobs: submodules: ${{inputs.submodules}} fetch-depth: ${{inputs.checkout_depth}} filter: tree:0 + - name: Debug Info + uses: ./.github/actions/debug - name: Set build envs run: | cat >> "$GITHUB_ENV" << 'EOF' From b55d0b54ea2810355723fb2edb0705fc07f4c320 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 13 Sep 2024 17:27:31 +0200 Subject: [PATCH 134/148] Merge steps together to minimize grouping --- .github/actions/debug/action.yml | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/actions/debug/action.yml b/.github/actions/debug/action.yml index d116c422e03..b45465809d2 100644 --- a/.github/actions/debug/action.yml +++ b/.github/actions/debug/action.yml @@ -4,35 +4,31 @@ description: Prints workflow debug info runs: using: "composite" steps: - - name: Print envs + - name: Envs, event.json and contexts shell: bash run: | - echo "::group::Envs" + echo '::group::Environment variables' env | sort - echo "::endgroup::" - - name: Print Event.json - shell: bash - run: | - echo "::group::Event.json" + echo '::endgroup::' + + echo '::group::event.json' python3 -m json.tool "$GITHUB_EVENT_PATH" - echo "::endgroup::" - - name: Pring contexts - shell: bash - run: | + echo '::endgroup::' + cat << 'EOF' - ::group::github + ::group::github context ${{ toJSON(github) }} ::endgroup:: - ::group::runner - ${{ toJSON(runner) }} - ::endgroup:: - - ::group::env + ::group::env context ${{ toJSON(env) }} ::endgroup:: - ::group::job + ::group::runner context + ${{ toJSON(runner) }} + ::endgroup:: + + ::group::job context ${{ toJSON(job) }} ::endgroup:: EOF From 4225ba79f9307cfee9a367966049185e4c722c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=D1=81hael=20Stetsyuk?= <59827607+mstetsyuk@users.noreply.github.com> Date: Fri, 13 Sep 2024 17:35:11 +0100 Subject: [PATCH 135/148] use base/sleep.h --- src/Databases/DatabaseReplicatedWorker.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 07150957912..7993a41df53 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include namespace fs = std::filesystem; @@ -251,7 +251,7 @@ String DatabaseReplicatedDDLWorker::enqueueQueryImpl(const ZooKeeperPtr & zookee else if (code != Coordination::Error::ZNODEEXISTS) zkutil::KeeperMultiException::check(code, ops, res); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); + sleepForMilliseconds(50); } if (counter_path.empty()) From a461d20af92278fb973ba7a86ec84fb58946ce74 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 13 Sep 2024 23:03:01 +0200 Subject: [PATCH 136/148] Masking sensitive info in gcs() table function. --- src/Parsers/FunctionSecretArgumentsFinderAST.h | 3 ++- tests/integration/test_mask_sensitive_info/test.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Parsers/FunctionSecretArgumentsFinderAST.h b/src/Parsers/FunctionSecretArgumentsFinderAST.h index 94da30922cc..15d9a8d5add 100644 --- a/src/Parsers/FunctionSecretArgumentsFinderAST.h +++ b/src/Parsers/FunctionSecretArgumentsFinderAST.h @@ -74,7 +74,8 @@ private: findMySQLFunctionSecretArguments(); } else if ((function.name == "s3") || (function.name == "cosn") || (function.name == "oss") || - (function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg")) + (function.name == "deltaLake") || (function.name == "hudi") || (function.name == "iceberg") || + (function.name == "gcs")) { /// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...) findS3FunctionSecretArguments(/* is_cluster_function= */ false); diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 8d5345082ff..5366de39ea7 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -393,6 +393,7 @@ def test_table_functions(): f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, connection_string = '{azure_conn_string}', container = 'cont', blob_path = 'test_simple_16.csv', format = 'CSV')", f"azureBlobStorageCluster('test_shard_localhost', named_collection_2, storage_account_url = '{azure_storage_account_url}', container = 'cont', blob_path = 'test_simple_17.csv', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"iceberg('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')", + f"gcs('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')", ] def make_test_case(i): From 5f464a6d74a2a50b761b55b49479c50afe99dc66 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Sun, 15 Sep 2024 07:09:49 -0300 Subject: [PATCH 137/148] trigger ic From 37411bf240e7c94a0e9e07c645a7ec74d6758aab Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 15 Sep 2024 15:06:14 +0000 Subject: [PATCH 138/148] Fix sizing with unconstrained thread pool size --- src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp index 641770b16e9..bf9aad6545d 100644 --- a/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexVectorSimilarity.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -270,6 +271,8 @@ void updateImpl(const ColumnArray * column_array, const ColumnArray::Offsets & c /// Reserving space is mandatory size_t max_thread_pool_size = Context::getGlobalContextInstance()->getServerSettings().max_build_vector_similarity_index_thread_pool_size; + if (max_thread_pool_size == 0) + max_thread_pool_size = getNumberOfPhysicalCPUCores(); unum::usearch::index_limits_t limits(roundUpToPowerOfTwoOrZero(index->size() + rows), max_thread_pool_size); index->reserve(limits); From 14feba8443360fb7c809415609a2b3865f188f48 Mon Sep 17 00:00:00 2001 From: Oleksandr Date: Mon, 16 Sep 2024 11:42:09 +0300 Subject: [PATCH 139/148] Docs: Update index --- docs/en/engines/database-engines/index.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/en/engines/database-engines/index.md b/docs/en/engines/database-engines/index.md index 233cbbb4247..3743717198c 100644 --- a/docs/en/engines/database-engines/index.md +++ b/docs/en/engines/database-engines/index.md @@ -13,16 +13,17 @@ Here is a complete list of available database engines. Follow the links for more - [Atomic](../../engines/database-engines/atomic.md) -- [MySQL](../../engines/database-engines/mysql.md) - -- [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) - - [Lazy](../../engines/database-engines/lazy.md) -- [PostgreSQL](../../engines/database-engines/postgresql.md) - -- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) - - [Replicated](../../engines/database-engines/replicated.md) +- [PostgreSQL](../../engines/database-engines/postgresql.md) + +- [MySQL](../../engines/database-engines/mysql.md) + - [SQLite](../../engines/database-engines/sqlite.md) + +- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) + +- [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) + From 85af661b9c419284de3999cb5696356e07e5752a Mon Sep 17 00:00:00 2001 From: Oleksandr Date: Mon, 16 Sep 2024 12:57:24 +0300 Subject: [PATCH 140/148] Docs: Update index --- docs/en/engines/database-engines/index.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/engines/database-engines/index.md b/docs/en/engines/database-engines/index.md index 3743717198c..fffd780e62a 100644 --- a/docs/en/engines/database-engines/index.md +++ b/docs/en/engines/database-engines/index.md @@ -15,15 +15,15 @@ Here is a complete list of available database engines. Follow the links for more - [Lazy](../../engines/database-engines/lazy.md) -- [Replicated](../../engines/database-engines/replicated.md) +- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) + +- [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) + +- [MySQL](../../engines/database-engines/mysql.md) - [PostgreSQL](../../engines/database-engines/postgresql.md) - -- [MySQL](../../engines/database-engines/mysql.md) + +- [Replicated](../../engines/database-engines/replicated.md) - [SQLite](../../engines/database-engines/sqlite.md) -- [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) - -- [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) - From de85f5f251d52db319757d3c3324182edbbecde2 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:15:11 +0200 Subject: [PATCH 141/148] empty commit (I've changed the changelog entry) From 60f893bf8733e2ea130f99359d7c44ea6052681d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 16 Sep 2024 12:38:42 +0200 Subject: [PATCH 142/148] Update filelog.md --- docs/en/engines/table-engines/special/filelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/special/filelog.md b/docs/en/engines/table-engines/special/filelog.md index 82201053bc5..b9fb4ceaedf 100644 --- a/docs/en/engines/table-engines/special/filelog.md +++ b/docs/en/engines/table-engines/special/filelog.md @@ -97,7 +97,7 @@ If you want to change the target table by using `ALTER`, we recommend disabling - `_filename` - Name of the log file. Data type: `LowCardinality(String)`. - `_offset` - Offset in the log file. Data type: `UInt64`. -Additional virtual columns when `kafka_handle_error_mode='stream'`: +Additional virtual columns when `handle_error_mode='stream'`: - `_raw_record` - Raw record that couldn't be parsed successfully. Data type: `Nullable(String)`. - `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`. From f401eccc642c1581f1cd5d7a92e880ed373d8fc0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 16 Sep 2024 12:16:07 +0200 Subject: [PATCH 143/148] Fix Keeper multi request preprocessing with NOAUTH --- src/Coordination/KeeperStorage.cpp | 15 ++++- src/Coordination/KeeperStorage.h | 1 + src/Coordination/tests/gtest_coordination.cpp | 56 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 2eb9ab30efa..63dc39092cf 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -2545,6 +2545,7 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro response.responses[i]->error = failed_multi->error_codes[i]; } + response.error = failed_multi->global_error; storage.uncommitted_state.commit(zxid); return response_ptr; } @@ -2901,7 +2902,19 @@ void KeeperStorage::preprocessRequest( if (check_acl && !request_processor->checkAuth(*this, session_id, false)) { - uncommitted_state.deltas.emplace_back(new_last_zxid, Coordination::Error::ZNOAUTH); + /// Multi requests handle failures using FailedMultiDelta + if (zk_request->getOpNum() == Coordination::OpNum::Multi || zk_request->getOpNum() == Coordination::OpNum::MultiRead) + { + const auto & multi_request = dynamic_cast(*zk_request); + std::vector response_errors; + response_errors.resize(multi_request.requests.size(), Coordination::Error::ZOK); + uncommitted_state.deltas.emplace_back( + new_last_zxid, KeeperStorage::FailedMultiDelta{std::move(response_errors), Coordination::Error::ZNOAUTH}); + } + else + { + uncommitted_state.deltas.emplace_back(new_last_zxid, Coordination::Error::ZNOAUTH); + } return; } diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 904af76ef37..6fbc4c2b168 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -522,6 +522,7 @@ public: struct FailedMultiDelta { std::vector error_codes; + Coordination::Error global_error{Coordination::Error::ZOK}; }; // Denotes end of a subrequest in multi request diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 73402af5ec4..46f36fe0039 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -2280,6 +2280,62 @@ TYPED_TEST(CoordinationTest, TestPreprocessWhenCloseSessionIsPrecommitted) } } +TYPED_TEST(CoordinationTest, TestMultiRequestWithNoAuth) +{ + using namespace Coordination; + using namespace DB; + + ChangelogDirTest snapshots("./snapshots"); + this->setSnapshotDirectory("./snapshots"); + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + ResponsesQueue queue(std::numeric_limits::max()); + SnapshotsQueue snapshots_queue{1}; + int64_t session_without_auth = 1; + int64_t session_with_auth = 2; + size_t term = 0; + + auto state_machine = std::make_shared>(queue, snapshots_queue, this->keeper_context, nullptr); + state_machine->init(); + + auto & storage = state_machine->getStorageUnsafe(); + + auto auth_req = std::make_shared(); + auth_req->scheme = "digest"; + auth_req->data = "test_user:test_password"; + + // Add auth data to the session + auto auth_entry = getLogEntryFromZKRequest(term, session_with_auth, state_machine->getNextZxid(), auth_req); + state_machine->pre_commit(1, auth_entry->get_buf()); + state_machine->commit(1, auth_entry->get_buf()); + + std::string node_with_acl = "/node_with_acl"; + { + auto create_req = std::make_shared(); + create_req->path = node_with_acl; + create_req->data = "notmodified"; + create_req->acls = {{.permissions = ACL::Read, .scheme = "auth", .id = ""}}; + auto create_entry = getLogEntryFromZKRequest(term, session_with_auth, state_machine->getNextZxid(), create_req); + state_machine->pre_commit(3, create_entry->get_buf()); + state_machine->commit(3, create_entry->get_buf()); + ASSERT_TRUE(storage.container.contains(node_with_acl)); + } + Requests ops; + ops.push_back(zkutil::makeSetRequest(node_with_acl, "modified", -1)); + ops.push_back(zkutil::makeCheckRequest("/nonexistentnode", -1)); + auto multi_req = std::make_shared(ops, ACLs{}); + auto multi_entry = getLogEntryFromZKRequest(term, session_without_auth, state_machine->getNextZxid(), multi_req); + state_machine->pre_commit(4, multi_entry->get_buf()); + state_machine->commit(4, multi_entry->get_buf()); + + auto node_it = storage.container.find(node_with_acl); + ASSERT_FALSE(node_it == storage.container.end()); + ASSERT_TRUE(node_it->value.getData() == "notmodified"); +} + TYPED_TEST(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted) { using namespace Coordination; From 20b25566f5ac0d207e9eee1377f9e327bb0543b5 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 7 Sep 2024 16:20:20 +0000 Subject: [PATCH 144/148] Remove superfluous --multiquery/-n --- tests/docker_scripts/stateful_runner.sh | 2 +- tests/docker_scripts/stress_runner.sh | 2 +- tests/integration/helpers/client.py | 1 - tests/performance/scripts/compare.sh | 12 +-- ...006_client_test_hint_no_such_error_name.sh | 2 +- .../0_stateless/02030_rocksdb_race_long.sh | 6 +- .../02151_hash_table_sizes_stats_joins.sh | 2 +- .../02229_client_stop_multiquery_in_SIGINT.sh | 4 +- ...41_filesystem_cache_on_write_operations.sh | 8 +- .../0_stateless/02242_delete_user_race.sh | 2 +- .../0_stateless/02243_drop_user_grant_race.sh | 4 +- .../02266_protobuf_format_google_wrappers.sh | 16 ++-- .../02286_drop_filesystem_cache.sh | 6 +- .../02313_filesystem_cache_seeks.sh | 2 +- .../02340_parts_refcnt_mergetree.sh | 6 +- .../0_stateless/02344_describe_cache.sh | 4 +- tests/queries/0_stateless/02352_rwlock.sh | 4 +- .../0_stateless/02380_insert_mv_race.sh | 4 +- ...entelemetry_insert_on_distributed_table.sh | 8 +- .../02419_keeper_map_primary_key.sh | 2 +- .../02443_detach_attach_partition.sh | 2 +- .../0_stateless/02473_optimize_old_parts.sh | 8 +- .../0_stateless/02477_s3_request_throttler.sh | 2 +- ...02494_zero_copy_projection_cancel_fetch.sh | 4 +- .../02504_regexp_dictionary_ua_parser.sh | 8 +- .../02504_regexp_dictionary_yaml_source.sh | 18 ++-- .../02515_cleanup_async_insert_block_ids.sh | 6 +- .../02585_query_status_deadlock.sh | 2 +- ...de_client_info_initial_query_start_time.sh | 14 ++-- ...rofile_events_from_query_log_and_client.sh | 16 ++-- .../02676_optimize_old_parts_replicated.sh | 8 +- .../02686_postgres_protocol_decimal_256.sh | 2 +- .../0_stateless/02700_s3_part_INT_MAX.sh | 2 +- .../0_stateless/02725_start_stop_fetches.sh | 2 +- .../02731_zero_objects_in_metadata.sh | 6 +- tests/queries/0_stateless/02766_prql.sh | 2 +- .../0_stateless/02770_async_buffer_ignore.sh | 4 +- .../02789_filesystem_cache_alignment.sh | 18 ++-- ...89_reading_from_s3_with_connection_pool.sh | 8 +- .../0_stateless/02801_backup_native_copy.sh | 10 +-- .../0_stateless/02803_backup_tmp_files.sh | 2 +- ...2808_custom_disk_with_user_defined_name.sh | 8 +- .../02808_filesystem_cache_drop_query.sh | 22 ++--- ...ackup_use_same_password_for_base_backup.sh | 2 +- ...use_same_s3_credentials_for_base_backup.sh | 2 +- .../02844_max_backup_bandwidth_s3.sh | 6 +- .../0_stateless/02867_storage_set_tsan.sh | 4 +- .../02900_union_schema_inference_mode.sh | 10 +-- .../02908_many_requests_to_system_replicas.sh | 2 +- .../02922_deduplication_with_zero_copy.sh | 10 +-- .../02932_refreshable_materialized_views_1.sh | 72 ++++++++-------- .../02932_refreshable_materialized_views_2.sh | 82 +++++++++---------- .../0_stateless/02941_variant_type_1.sh | 10 +-- .../0_stateless/02941_variant_type_2.sh | 4 +- .../0_stateless/02941_variant_type_3.sh | 4 +- .../0_stateless/02941_variant_type_4.sh | 2 +- ...ynamically_change_filesystem_cache_size.sh | 10 +-- .../0_stateless/02960_polygon_bound_bug.sh | 2 +- .../02961_storage_config_volume_priority.sh | 4 +- ...02963_remote_read_small_buffer_size_bug.sh | 6 +- ...allel_replicas_join_algo_and_analyzer_1.sh | 2 +- ...allel_replicas_join_algo_and_analyzer_2.sh | 2 +- ...allel_replicas_join_algo_and_analyzer_3.sh | 2 +- .../02969_auto_format_detection.sh | 6 +- .../02980_s3_plain_DROP_TABLE_MergeTree.sh | 4 +- ...s3_plain_DROP_TABLE_ReplicatedMergeTree.sh | 6 +- ...ion_insert_several_blocks_nonreplicated.sh | 2 +- ...cation_insert_several_blocks_replicated.sh | 2 +- ..._generates_several_blocks_nonreplicated.sh | 2 +- ..._mv_generates_several_blocks_replicated.sh | 2 +- ...several_mv_into_one_table_nonreplicated.sh | 2 +- ...on_several_mv_into_one_table_replicated.sh | 2 +- .../0_stateless/03008_s3_plain_rewritable.sh | 8 +- ...2_dynamically_resize_filesystem_cache_2.sh | 6 +- .../03039_dynamic_aggregating_merge_tree.sh | 2 +- .../03039_dynamic_collapsing_merge_tree.sh | 2 +- .../03039_dynamic_replacing_merge_tree.sh | 2 +- .../03039_dynamic_summing_merge_tree.sh | 2 +- ...dynamic_versioned_collapsing_merge_tree.sh | 2 +- .../0_stateless/03151_unload_index_race.sh | 6 +- .../03172_error_log_table_not_empty.sh | 8 +- .../03203_hive_style_partitioning.sh | 18 ++-- ...kup_and_clear_old_temporary_directories.sh | 4 +- 83 files changed, 298 insertions(+), 299 deletions(-) diff --git a/tests/docker_scripts/stateful_runner.sh b/tests/docker_scripts/stateful_runner.sh index 86f6a299ad3..defc24781a1 100755 --- a/tests/docker_scripts/stateful_runner.sh +++ b/tests/docker_scripts/stateful_runner.sh @@ -105,7 +105,7 @@ setup_logs_replication clickhouse-client --query "SHOW DATABASES" clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql +clickhouse-client < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 039c60c8e4e..af93c93f202 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -62,7 +62,7 @@ start_server setup_logs_replication clickhouse-client --query "CREATE DATABASE datasets" -clickhouse-client --multiquery < /repo/tests/docker_scripts/create.sql +clickhouse-client < /repo/tests/docker_scripts/create.sql clickhouse-client --query "SHOW TABLES FROM datasets" clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test" diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 8ba7b342020..97405860784 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -89,7 +89,6 @@ class Client: command = self.command[:] if stdin is None: - command += ["--multiquery"] stdin = sql else: command += ["--query", sql] diff --git a/tests/performance/scripts/compare.sh b/tests/performance/scripts/compare.sh index da7bbf77a28..c3566c51a16 100755 --- a/tests/performance/scripts/compare.sh +++ b/tests/performance/scripts/compare.sh @@ -427,7 +427,7 @@ do done # for each query run, prepare array of metrics from query log -clickhouse-local --multiquery --query " +clickhouse-local --query " create view query_runs as select * from file('analyze/query-runs.tsv', TSV, 'test text, query_index int, query_id text, version UInt8, time float'); @@ -582,7 +582,7 @@ numactl --cpunodebind=all --membind=all numactl --show # If the available memory falls below 2 * size, GNU parallel will suspend some of the running jobs. numactl --cpunodebind=all --membind=all parallel -v --joblog analyze/parallel-log.txt --memsuspend 15G --null < analyze/commands.txt 2>> analyze/errors.log -clickhouse-local --multiquery --query " +clickhouse-local --query " -- Join the metric names back to the metric statistics we've calculated, and make -- a denormalized table of them -- statistics for all metrics for all queries. -- The WITH, ARRAY JOIN and CROSS JOIN do not like each other: @@ -680,7 +680,7 @@ rm ./*.{rep,svg} test-times.tsv test-dump.tsv unstable.tsv unstable-query-ids.ts cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: -clickhouse-local --multiquery --query " +clickhouse-local --query " create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -981,7 +981,7 @@ create table all_query_metrics_tsv engine File(TSV, 'report/all-query-metrics.ts for version in {right,left} do rm -rf data - clickhouse-local --multiquery --query " + clickhouse-local --query " create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1151,7 +1151,7 @@ function report_metrics rm -rf metrics ||: mkdir metrics -clickhouse-local --multiquery --query " +clickhouse-local --query " create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes) ; @@ -1211,7 +1211,7 @@ function upload_results # Prepare info for the CI checks table. rm -f ci-checks.tsv - clickhouse-local --multiquery --query " + clickhouse-local --query " create view queries as select * from file('report/queries.tsv', TSVWithNamesAndTypes); create table ci_checks engine File(TSVWithNamesAndTypes, 'ci-checks.tsv') diff --git a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh index 972ff3ba73f..7bf8100a53d 100755 --- a/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh +++ b/tests/queries/0_stateless/02006_client_test_hint_no_such_error_name.sh @@ -5,4 +5,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' +$CLICKHOUSE_CLIENT -q 'select 1 -- { clientError FOOBAR }' |& grep -o 'No error code with name:.*' diff --git a/tests/queries/0_stateless/02030_rocksdb_race_long.sh b/tests/queries/0_stateless/02030_rocksdb_race_long.sh index da31861991c..7d329f78d98 100755 --- a/tests/queries/0_stateless/02030_rocksdb_race_long.sh +++ b/tests/queries/0_stateless/02030_rocksdb_race_long.sh @@ -12,14 +12,14 @@ echo " DROP TABLE IF EXISTS rocksdb_race; CREATE TABLE rocksdb_race (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key); INSERT INTO rocksdb_race SELECT '1_' || toString(number), number FROM numbers(100000); -" | $CLICKHOUSE_CLIENT -n +" | $CLICKHOUSE_CLIENT function read_stat_thread() { while true; do echo " SELECT * FROM system.rocksdb FORMAT Null; - " | $CLICKHOUSE_CLIENT -n + " | $CLICKHOUSE_CLIENT done } @@ -29,7 +29,7 @@ function truncate_thread() sleep 3s; echo " TRUNCATE TABLE rocksdb_race; - " | $CLICKHOUSE_CLIENT -n + " | $CLICKHOUSE_CLIENT done } diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh index 007dae6e427..a9b5c461c5a 100755 --- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh +++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh @@ -12,7 +12,7 @@ opts=( --join_algorithm='parallel_hash' ) -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " CREATE TABLE t1(a UInt32, b UInt32) ENGINE=MergeTree ORDER BY (); INSERT INTO t1 SELECT number, number FROM numbers_mt(1e6); diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh index e5d00bc1a1c..b23164f8354 100755 --- a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -5,12 +5,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -nm -q " +timeout -s INT 3s $CLICKHOUSE_CLIENT --max_block_size 1 -m -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " -timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -nm -q " +timeout -s INT 3s $CLICKHOUSE_LOCAL --max_block_size 1 -m -q " SELECT sleep(1) FROM numbers(100) FORMAT Null; SELECT 'FAIL'; " diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh index f8e7b7e7e72..8a44b65e6d1 100755 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh @@ -16,7 +16,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM DROP FILESYSTEM CACHE" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -37,7 +37,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -70,7 +70,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --enable_filesystem_cache_on_write_operations=1 --query "INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)" - $CLICKHOUSE_CLIENT --echo -n --query "SELECT file_segment_range_begin, file_segment_range_end, size, state + $CLICKHOUSE_CLIENT --echo --query "SELECT file_segment_range_begin, file_segment_range_end, size, state FROM ( SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path @@ -109,7 +109,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --echo --query "SYSTEM FLUSH LOGS" - $CLICKHOUSE_CLIENT -n --query "SELECT + $CLICKHOUSE_CLIENT --query "SELECT query, ProfileEvents['RemoteFSReadBytes'] > 0 as remote_fs_read FROM system.query_log diff --git a/tests/queries/0_stateless/02242_delete_user_race.sh b/tests/queries/0_stateless/02242_delete_user_race.sh index 2af54276469..3cb79d69955 100755 --- a/tests/queries/0_stateless/02242_delete_user_race.sh +++ b/tests/queries/0_stateless/02242_delete_user_race.sh @@ -15,7 +15,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP ROLE IF EXISTS test_role_02242; CREATE ROLE test_role_02242; " diff --git a/tests/queries/0_stateless/02243_drop_user_grant_race.sh b/tests/queries/0_stateless/02243_drop_user_grant_race.sh index 4dce8e8124c..1972ecade06 100755 --- a/tests/queries/0_stateless/02243_drop_user_grant_race.sh +++ b/tests/queries/0_stateless/02243_drop_user_grant_race.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP ROLE IF EXISTS test_role_02244; CREATE ROLE test_role_02244; DROP USER IF EXISTS kek_02243; @@ -37,4 +37,4 @@ $CLICKHOUSE_CLIENT --user kek_02243 -q "SELECT * FROM test" 2>&1| grep -Fa "Exce $CLICKHOUSE_CLIENT -q "DROP ROLE IF EXISTS test_role_02243" $CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS test_user_02243" -$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" \ No newline at end of file +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS kek_02243" diff --git a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh index ae2a2351c6b..eb1109aaac1 100755 --- a/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh +++ b/tests/queries/0_stateless/02266_protobuf_format_google_wrappers.sh @@ -44,7 +44,7 @@ protobuf_info() { fi } -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS $MAIN_TABLE; DROP TABLE IF EXISTS $ROUNDTRIP_TABLE; DROP TABLE IF EXISTS $COMPATIBILITY_TABLE; @@ -78,14 +78,14 @@ echo $SET_OUTPUT echo echo "Insert $INITIAL_INSERT_VALUES into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " INSERT INTO $MAIN_TABLE VALUES $INITIAL_INSERT_VALUES; SELECT * FROM $MAIN_TABLE; " echo echo "Protobuf representation of the second row:" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MAIN_TABLE WHERE ref = 2 LIMIT 1 $(protobuf_info output ProtobufSingle Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -101,12 +101,12 @@ hexdump -C $MESSAGE_FILE_PATH echo echo "Insert proto message into table (Nullable(String), Int32):" -$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $ROUNDTRIP_TABLE $(protobuf_info input Protobuf Message)" < "$MESSAGE_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $ROUNDTRIP_TABLE" echo echo "Proto output of the table using Google wrapper:" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $ROUNDTRIP_TABLE $(protobuf_info output Protobuf Message)" > "$BINARY_FILE_PATH" hexdump -C $BINARY_FILE_PATH echo @@ -124,14 +124,14 @@ echo echo "Insert $MULTI_WRAPPER_VALUES and reinsert using Google wrappers into:" echo "Table (Nullable(Int32), Nullable(Int32), Int32):" $CLICKHOUSE_CLIENT --query "INSERT INTO $MULTI_TABLE VALUES $MULTI_WRAPPER_VALUES" -$CLICKHOUSE_CLIENT -n --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" -$CLICKHOUSE_CLIENT -n --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_OUTPUT SELECT * FROM $MULTI_TABLE $(protobuf_info output Protobuf MessageMultiWrapper)" > "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "$SET_INPUT INSERT INTO $MULTI_TABLE $(protobuf_info input Protobuf MessageMultiWrapper)" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM $MULTI_TABLE" rm "$BINARY_FILE_PATH" rm "$MESSAGE_FILE_PATH" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE $MAIN_TABLE; DROP TABLE $ROUNDTRIP_TABLE; DROP TABLE $COMPATIBILITY_TABLE; diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index dab777fcc31..672bf4d068b 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -11,7 +11,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286" - $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_02286 (key UInt32, value String) + $CLICKHOUSE_CLIENT --query "CREATE TABLE test_02286 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='$STORAGE_POLICY', min_bytes_for_wide_part = 10485760" @@ -38,7 +38,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null" $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT -n --query "SELECT count() + $CLICKHOUSE_CLIENT --query "SELECT count() FROM ( SELECT arrayJoin(cache_paths) AS cache_path, @@ -54,7 +54,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" $CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache" - $CLICKHOUSE_CLIENT -n --query "SELECT cache_path, local_path + $CLICKHOUSE_CLIENT --query "SELECT cache_path, local_path FROM ( SELECT arrayJoin(cache_paths) AS cache_path, diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh index b7adde6fcbb..64fae06f220 100755 --- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh +++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh @@ -23,7 +23,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d ORDER BY tuple() SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null - $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 -n --query "INSERT INTO test_02313 + $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02313 SELECT * FROM generateRandom('id Int32, val String') LIMIT 100000" diff --git a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh index bd018018789..f252b9304cd 100755 --- a/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh +++ b/tests/queries/0_stateless/02340_parts_refcnt_mergetree.sh @@ -9,7 +9,7 @@ function check_refcnt_for_table() { local table=$1 && shift - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " system stop merges $table; -- cleanup thread may hold the parts lock system stop cleanup $table; @@ -66,14 +66,14 @@ function check_refcnt_for_table() # NOTE: index_granularity=1 to cancel ASAP -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " drop table if exists data_02340; create table data_02340 (key Int, part Int) engine=MergeTree() partition by part order by key settings index_granularity=1; " || exit 1 check_refcnt_for_table data_02340 $CLICKHOUSE_CLIENT -q "drop table data_02340 sync" -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " drop table if exists data_02340_rep sync; create table data_02340_rep (key Int, part Int) engine=ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '1') partition by part order by key settings index_granularity=1; " || exit 1 diff --git a/tests/queries/0_stateless/02344_describe_cache.sh b/tests/queries/0_stateless/02344_describe_cache.sh index c5373b4d7e3..4cc4c415f3f 100755 --- a/tests/queries/0_stateless/02344_describe_cache.sh +++ b/tests/queries/0_stateless/02344_describe_cache.sh @@ -7,14 +7,14 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02344_describe_cache_test" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = '$disk_name', disk = 's3_disk', load_metadata_asynchronously = 0); """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02352_rwlock.sh b/tests/queries/0_stateless/02352_rwlock.sh index b4a77e0b08a..7f02b3ee935 100755 --- a/tests/queries/0_stateless/02352_rwlock.sh +++ b/tests/queries/0_stateless/02352_rwlock.sh @@ -24,7 +24,7 @@ function wait_query_by_id_started() # wait for query to be started while [ "$($CLICKHOUSE_CLIENT "$@" -q "select count() from system.processes where query_id = '$query_id'")" -ne 1 ]; do if [ "$( - $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -nm -q " + $CLICKHOUSE_CLIENT --max_bytes_before_external_group_by 0 -m -q " system flush logs; select count() from system.query_log @@ -52,7 +52,7 @@ $CLICKHOUSE_CLIENT -q "CREATE DATABASE ${CLICKHOUSE_DATABASE}_ordinary Engine=Or # debug build on CI, so if this will happen, then DROP query will be # finished instantly, and to avoid flakiness we will retry in this case while :; do - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}_ordinary.data_02352; CREATE TABLE ${CLICKHOUSE_DATABASE}_ordinary.data_02352 (key Int) Engine=Null(); " diff --git a/tests/queries/0_stateless/02380_insert_mv_race.sh b/tests/queries/0_stateless/02380_insert_mv_race.sh index 725c7eacce6..a5f05365180 100755 --- a/tests/queries/0_stateless/02380_insert_mv_race.sh +++ b/tests/queries/0_stateless/02380_insert_mv_race.sh @@ -9,13 +9,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q "ATTACH TABLE mv" |& { +$CLICKHOUSE_CLIENT -m -q "ATTACH TABLE mv" |& { # CANNOT_GET_CREATE_TABLE_QUERY -- ATTACH TABLE IF EXISTS # TABLE_ALREADY_EXISTS -- ATTACH TABLE IF NOT EXISTS grep -F -m1 Exception | grep -v -e CANNOT_GET_CREATE_TABLE_QUERY -e TABLE_ALREADY_EXISTS } -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS null; CREATE TABLE null (key Int) ENGINE = Null; DROP TABLE IF EXISTS mv; diff --git a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh index c692c37f27f..9e8b854375b 100755 --- a/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh +++ b/tests/queries/0_stateless/02417_opentelemetry_insert_on_distributed_table.sh @@ -27,7 +27,7 @@ function insert() function check_span() { -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS; SELECT operation_name, @@ -50,7 +50,7 @@ ${CLICKHOUSE_CLIENT} -nq " # $2 - value of distributed_foreground_insert function check_span_kind() { -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " SYSTEM FLUSH LOGS; SELECT count() @@ -65,7 +65,7 @@ ${CLICKHOUSE_CLIENT} -nq " # # Prepare tables for tests # -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.local_opentelemetry; @@ -122,7 +122,7 @@ check_span_kind $trace_id 'CLIENT' # # Cleanup # -${CLICKHOUSE_CLIENT} -nq " +${CLICKHOUSE_CLIENT} -q " DROP TABLE ${CLICKHOUSE_DATABASE}.dist_opentelemetry; DROP TABLE ${CLICKHOUSE_DATABASE}.local_opentelemetry; " diff --git a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh index c43c5bb6408..6a23f666f36 100755 --- a/tests/queries/0_stateless/02419_keeper_map_primary_key.sh +++ b/tests/queries/0_stateless/02419_keeper_map_primary_key.sh @@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02419_test SYNC;" test_primary_key() { - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " CREATE TABLE 02419_test (key UInt64, value Float64) Engine=KeeperMap('/' || currentDatabase() || '/test2418', 3) PRIMARY KEY($1); INSERT INTO 02419_test VALUES (1, 1.1), (2, 2.2); SELECT value FROM 02419_test WHERE key = 1; diff --git a/tests/queries/0_stateless/02443_detach_attach_partition.sh b/tests/queries/0_stateless/02443_detach_attach_partition.sh index 6a47b7d8d61..4a5377a952a 100755 --- a/tests/queries/0_stateless/02443_detach_attach_partition.sh +++ b/tests/queries/0_stateless/02443_detach_attach_partition.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/replication.lib -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; diff --git a/tests/queries/0_stateless/02473_optimize_old_parts.sh b/tests/queries/0_stateless/02473_optimize_old_parts.sh index b563bc31b39..3a4e6145f12 100755 --- a/tests/queries/0_stateless/02473_optimize_old_parts.sh +++ b/tests/queries/0_stateless/02473_optimize_old_parts.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_with_merge; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_without_merge; SELECT 'With merge any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_with_merge; SELECT 'With merge partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_with_merge SELECT 3;" wait_for_number_of_parts 'test_with_merge' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_with_merge' AND active; diff --git a/tests/queries/0_stateless/02477_s3_request_throttler.sh b/tests/queries/0_stateless/02477_s3_request_throttler.sh index c74cb598d42..5e6185e7304 100755 --- a/tests/queries/0_stateless/02477_s3_request_throttler.sh +++ b/tests/queries/0_stateless/02477_s3_request_throttler.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " -- Limit S3 PUT request per second rate SET s3_max_put_rps = 2; SET s3_max_put_burst = 1; diff --git a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh index b72c3eb56c7..122684abe9c 100755 --- a/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh +++ b/tests/queries/0_stateless/02494_zero_copy_projection_cancel_fetch.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " @@ -60,7 +60,7 @@ wait $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat1 WHERE NOT ignore(*)" $CLICKHOUSE_CLIENT --query "SELECT count() FROM wikistat2 WHERE NOT ignore(*)" -$CLICKHOUSE_CLIENT -n --query " +$CLICKHOUSE_CLIENT --query " DROP TABLE IF EXISTS wikistat1 SYNC; DROP TABLE IF EXISTS wikistat2 SYNC; " diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh index e389cf410e8..77c8049bd58 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -11,7 +11,7 @@ cp $CURDIR/data_ua_parser/os.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/browser.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ cp $CURDIR/data_ua_parser/device.yaml ${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}/ -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; @@ -61,10 +61,10 @@ create table user_agents Engine = Log(); " -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" select ua, device, concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser , concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os @@ -74,7 +74,7 @@ from ( dictGet('regexp_device', 'device_replacement', ua) device from user_agents) order by ua; " -$CLICKHOUSE_CLIENT -n --query=" +$CLICKHOUSE_CLIENT --query=" drop dictionary if exists regexp_os; drop dictionary if exists regexp_browser; drop dictionary if exists regexp_device; diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh index 68a87a14320..dec03bcaaa0 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh @@ -27,7 +27,7 @@ cat > "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < "$yaml" < /dev/null & diff --git a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh index 3b0d2309784..9f743581e1a 100755 --- a/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh +++ b/tests/queries/0_stateless/02590_interserver_mode_client_info_initial_query_start_time.sh @@ -13,7 +13,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function get_query_id() { random_str 10; } -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists buf; drop table if exists dist; drop table if exists data; @@ -31,7 +31,7 @@ query_id="$(get_query_id)" # test, since we care about the difference between NOW() and there should # not be any significant difference. $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " @@ -42,25 +42,25 @@ query_id="$(get_query_id)" # this query (and all subsequent) should reuse the previous connection (at least most of the time) $CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -q "select * from dist" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; " echo "INSERT" query_id="$(get_query_id)" -$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -nm -q " +$CLICKHOUSE_CLIENT --prefer_localhost_replica=0 --query_id "$query_id" -m -q " insert into dist_dist values (1),(2); select * from data; " sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist_dist" +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist_dist" sleep 1 -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "system flush distributed dist" +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q "system flush distributed dist" echo "CHECK" -$CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " +$CLICKHOUSE_CLIENT -m --param_query_id "$query_id" -q " select * from data order by key; system flush logs; select count(), count(distinct initial_query_start_time_microseconds) from system.query_log where type = 'QueryFinish' and initial_query_id = {query_id:String}; diff --git a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh index ff534a6a2e6..5015521ca4d 100755 --- a/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh +++ b/tests/queries/0_stateless/02675_profile_events_from_query_log_and_client.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh echo "INSERT TO S3" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO TABLE FUNCTION s3('http://localhost:11111/test/profile_events.csv', 'test', 'testtest', 'CSV', 'number UInt64') SELECT number FROM numbers(1000000) SETTINGS s3_max_single_part_upload_size = 10, s3_truncate_on_insert = 1; " 2>&1 | $CLICKHOUSE_LOCAL -q " WITH '(\\w+): (\\d+)' AS pattern, @@ -30,7 +30,7 @@ SELECT * FROM ( " echo "CHECK WITH query_log" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; SELECT type, 'S3CreateMultipartUpload', ProfileEvents['S3CreateMultipartUpload'], @@ -45,7 +45,7 @@ ORDER BY query_start_time DESC; " echo "CREATE" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS times; CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t SETTINGS @@ -56,29 +56,29 @@ CREATE TABLE times (t DateTime) ENGINE MergeTree ORDER BY t " echo "INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO times SELECT now() + INTERVAL 1 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "READ" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " SELECT '1', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "INSERT and READ INSERT" -$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -nq " +$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q " INSERT INTO times SELECT now() + INTERVAL 2 day SETTINGS optimize_on_insert = 0; SELECT '2', min(t) FROM times SETTINGS optimize_use_implicit_projections = 1; INSERT INTO times SELECT now() + INTERVAL 3 day SETTINGS optimize_on_insert = 0; " 2>&1 | grep -o -e ' \[ .* \] FileOpen: .* ' echo "DROP" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " DROP TABLE times; " echo "CHECK with query_log" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; SELECT type, query, diff --git a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh index c1f28f9f079..84be5959344 100755 --- a/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh +++ b/tests/queries/0_stateless/02676_optimize_old_parts_replicated.sh @@ -21,7 +21,7 @@ wait_for_number_of_parts() { echo "$res" } -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE IF EXISTS test_without_merge; DROP TABLE IF EXISTS test_replicated; @@ -34,7 +34,7 @@ INSERT INTO test_without_merge SELECT 3;" wait_for_number_of_parts 'test_without_merge' 1 10 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_without_merge; SELECT 'With merge replicated any part range'; @@ -47,7 +47,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " DROP TABLE test_replicated; SELECT 'With merge replicated partition only'; @@ -60,7 +60,7 @@ INSERT INTO test_replicated SELECT 3;" wait_for_number_of_parts 'test_replicated' 1 100 -$CLICKHOUSE_CLIENT -nmq " +$CLICKHOUSE_CLIENT -mq " SELECT sleepEachRow(1) FROM numbers(9) SETTINGS function_sleep_max_microseconds_per_block = 10000000 FORMAT Null; -- Sleep for 9 seconds and verify that we keep the old part because it's the only one SELECT (now() - modification_time) > 5 FROM system.parts WHERE database = currentDatabase() AND table='test_replicated' AND active; diff --git a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh index 2a94f940327..a540a0981b3 100755 --- a/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh +++ b/tests/queries/0_stateless/02686_postgres_protocol_decimal_256.sh @@ -9,6 +9,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) echo " DROP USER IF EXISTS postgresql_user; CREATE USER postgresql_user HOST IP '127.0.0.1' IDENTIFIED WITH no_password; -" | $CLICKHOUSE_CLIENT -n +" | $CLICKHOUSE_CLIENT psql --host localhost --port ${CLICKHOUSE_PORT_POSTGRESQL} ${CLICKHOUSE_DATABASE} --user postgresql_user -c "SELECT 1.23::Decimal256(70) AS test;" diff --git a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh index cfb38c60615..dcd83f9fec3 100755 --- a/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh +++ b/tests/queries/0_stateless/02700_s3_part_INT_MAX.sh @@ -12,7 +12,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # too slow with this. # # Unfortunately, the test has to buffer it in memory. -$CLICKHOUSE_CLIENT --max_memory_usage 16G -nm -q " +$CLICKHOUSE_CLIENT --max_memory_usage 16G -m -q " INSERT INTO FUNCTION s3('http://localhost:11111/test/$CLICKHOUSE_DATABASE/test_INT_MAX.tsv', '', '', 'TSV') SELECT repeat('a', 1024) FROM numbers((pow(2, 30) * 2) / 1024) SETTINGS s3_max_single_part_upload_size = '5Gi'; diff --git a/tests/queries/0_stateless/02725_start_stop_fetches.sh b/tests/queries/0_stateless/02725_start_stop_fetches.sh index c9922455d94..604d0774b83 100755 --- a/tests/queries/0_stateless/02725_start_stop_fetches.sh +++ b/tests/queries/0_stateless/02725_start_stop_fetches.sh @@ -10,7 +10,7 @@ set -e NUM_REPLICAS=5 for i in $(seq 1 $NUM_REPLICAS); do - $CLICKHOUSE_CLIENT -n -q " + $CLICKHOUSE_CLIENT -q " DROP TABLE IF EXISTS r$i SYNC; CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1; " diff --git a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh index 78659b70129..6e40495157b 100755 --- a/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh +++ b/tests/queries/0_stateless/02731_zero_objects_in_metadata.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for DISK in s3_disk s3_cache do - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " DROP TABLE IF EXISTS test; CREATE TABLE test (id Int32, empty Array(Int32)) ENGINE=MergeTree ORDER BY id @@ -17,13 +17,13 @@ do SELECT * FROM test; " - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " BACKUP TABLE test TO Disk('backups', 'test_s3_backup'); DROP TABLE test; RESTORE TABLE test FROM Disk('backups', 'test_s3_backup'); " &>/dev/null - ${CLICKHOUSE_CLIENT} -n --query " + ${CLICKHOUSE_CLIENT} --query " SELECT * FROM test; SELECT empty FROM test; " diff --git a/tests/queries/0_stateless/02766_prql.sh b/tests/queries/0_stateless/02766_prql.sh index 85b1167027c..d1e4dca070c 100755 --- a/tests/queries/0_stateless/02766_prql.sh +++ b/tests/queries/0_stateless/02766_prql.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -n -q " +$CLICKHOUSE_CLIENT -q " CREATE TEMPORARY TABLE IF NOT EXISTS aboba ( user_id UInt32, diff --git a/tests/queries/0_stateless/02770_async_buffer_ignore.sh b/tests/queries/0_stateless/02770_async_buffer_ignore.sh index 37f002767d6..af9f18b232f 100755 --- a/tests/queries/0_stateless/02770_async_buffer_ignore.sh +++ b/tests/queries/0_stateless/02770_async_buffer_ignore.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -17,7 +17,7 @@ INSERT INTO test_s3 SELECT number, number FROM numbers(1000000); query="SELECT sum(b) FROM test_s3 WHERE a >= 100000 AND a <= 102000" query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['S3ReadRequestsCount'], ProfileEvents['ReadBufferFromS3Bytes'], diff --git a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh index c69c635f6ed..53d2832c589 100755 --- a/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh +++ b/tests/queries/0_stateless/02789_filesystem_cache_alignment.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() @@ -22,7 +22,7 @@ INSERT INTO test SELECT number, randomString(100) FROM numbers(1000000); " QUERY_ID=$RANDOM -$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -nm -q " +$CLICKHOUSE_CLIENT --query_id "$QUERY_ID" -m -q " SET enable_filesystem_cache_log = 1; SYSTEM DROP FILESYSTEM CACHE; SELECT * FROM test WHERE NOT ignore() LIMIT 1 FORMAT Null; @@ -49,14 +49,14 @@ WHERE query_id = '$QUERY_ID' " # File segments cannot be less that 20Mi, # except for last file segment in a file or if file size is less. -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size AND file_segment_size < 20 * 1024 * 1024; " -all=$($CLICKHOUSE_CLIENT -nm -q " +all=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size; ") @@ -68,7 +68,7 @@ else echo "FAIL" fi -count=$($CLICKHOUSE_CLIENT -nm -q " +count=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query) WHERE file_segment_size < file_size AND end_offset + 1 != file_size @@ -87,21 +87,21 @@ FROM (SELECT * FROM ($query)) AS cache_log INNER JOIN system.filesystem_cache AS cache ON cache_log.cache_path = cache.cache_path " -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND downloaded_size < 20 * 1024 * 1024; " -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size AND formatReadableSize(downloaded_size) not in ('20.00 MiB', '40.00 MiB'); " -all=$($CLICKHOUSE_CLIENT -nm -q " +all=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_size < file_size AND file_segment_range_end + 1 != file_size; ") @@ -112,7 +112,7 @@ else echo "FAIL" fi -count2=$($CLICKHOUSE_CLIENT -nm -q " +count2=$($CLICKHOUSE_CLIENT -m -q " SELECT count() FROM ($query2) WHERE file_segment_range_begin - file_segment_range_end + 1 < file_size AND file_segment_range_end + 1 != file_size diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh index 5a37d51233d..239485ab8dd 100755 --- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh +++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS test_s3; CREATE TABLE test_s3 (a UInt64, b UInt64) @@ -25,7 +25,7 @@ do query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -nm --query " + RES=$(${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['DiskConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' @@ -41,7 +41,7 @@ done while true do - query_id=$(${CLICKHOUSE_CLIENT} -nq " + query_id=$(${CLICKHOUSE_CLIENT} -q " create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n; set insert_keeper_fault_injection_probability=0; insert into mut values (1, 2, 3), (10, 20, 30); @@ -60,7 +60,7 @@ do ) limit 1 settings max_threads=1; " 2>&1) ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" - RES=$(${CLICKHOUSE_CLIENT} -nm --query " + RES=$(${CLICKHOUSE_CLIENT} -m --query " SELECT ProfileEvents['StorageConnectionsPreserved'] > 0 FROM system.query_log WHERE type = 'QueryFinish' diff --git a/tests/queries/0_stateless/02801_backup_native_copy.sh b/tests/queries/0_stateless/02801_backup_native_copy.sh index b8ee97a7c7d..bc485f903bc 100755 --- a/tests/queries/0_stateless/02801_backup_native_copy.sh +++ b/tests/queries/0_stateless/02801_backup_native_copy.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); @@ -16,28 +16,28 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "BACKUP TABLE data TO S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_native_copy') SETTINGS allow_s3_native_copy=true" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --format Null --query_id $query_id -q "RESTORE TABLE data AS data_no_native_copy FROM S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data_no_native_copy') SETTINGS allow_s3_native_copy=false" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT query, ProfileEvents['S3CopyObject']>0 FROM system.query_log WHERE type = 'QueryFinish' AND event_date >= yesterday() AND current_database = '$CLICKHOUSE_DATABASE' AND query_id = '$query_id' " diff --git a/tests/queries/0_stateless/02803_backup_tmp_files.sh b/tests/queries/0_stateless/02803_backup_tmp_files.sh index d86beae4923..2208d3d32ef 100755 --- a/tests/queries/0_stateless/02803_backup_tmp_files.sh +++ b/tests/queries/0_stateless/02803_backup_tmp_files.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) set -e -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple() settings disk='s3_disk'; insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh index 63fa60bd548..4d7ab4063be 100755 --- a/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh +++ b/tests/queries/0_stateless/02808_custom_disk_with_user_defined_name.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -17,17 +17,17 @@ SETTINGS disk = disk(name = 's3_disk', type = cache, max_size = '100Ki', path = disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() SETTINGS disk = disk(name = '$disk_name', type = cache, max_size = '100Ki', path = ${CLICKHOUSE_TEST_UNIQUE_NAME}, disk = s3_disk); """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.disks WHERE name = '$disk_name' """ diff --git a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh index 8a4a2e906b0..a7dfa035a22 100755 --- a/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh +++ b/tests/queries/0_stateless/02808_filesystem_cache_drop_query.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="${CLICKHOUSE_TEST_UNIQUE_NAME}" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, b String) ENGINE = MergeTree() ORDER BY tuple() @@ -22,29 +22,29 @@ query_id=$RANDOM $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Null SETTINGS enable_filesystem_cache_log = 1" -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY kek; """ 2>&1 | grep -q "Invalid cache key hex: kek" && echo "OK" || echo "FAIL" ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -nm --query """ +key=$($CLICKHOUSE_CLIENT -m --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -offset=$($CLICKHOUSE_CLIENT -nm --query """ +offset=$($CLICKHOUSE_CLIENT -m --query """ SELECT offset FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key OFFSET $offset; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key' AND file_segment_range_begin = $offset; """ @@ -54,18 +54,18 @@ $CLICKHOUSE_CLIENT --query_id "$query_id" --query "SELECT * FROM test FORMAT Nul ${CLICKHOUSE_CLIENT} -q " system flush logs" -key=$($CLICKHOUSE_CLIENT -nm --query """ +key=$($CLICKHOUSE_CLIENT -m --query """ SELECT key FROM system.filesystem_cache_log WHERE query_id = '$query_id' ORDER BY size DESC LIMIT 1; """) -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SYSTEM DROP FILESYSTEM CACHE '$disk_name' KEY $key """ -$CLICKHOUSE_CLIENT -nm --query """ +$CLICKHOUSE_CLIENT -m --query """ SELECT count() FROM system.filesystem_cache WHERE key = '$key'; """ diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh index a2b1a953e24..ecd73bcc8a0 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " DROP TABLE IF EXISTS data; DROP TABLE IF EXISTS data_1; DROP TABLE IF EXISTS data_2; diff --git a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh index 16ac095312c..e6a1d547945 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by tuple(); insert into data select * from numbers(10); diff --git a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh index 4650415c202..830e78e1e9f 100755 --- a/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh +++ b/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.sh @@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, disk='s3_disk'; -- reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds @@ -15,7 +15,7 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup2') SETTINGS allow_s3_native_copy=1" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT 'native_copy', @@ -26,7 +26,7 @@ $CLICKHOUSE_CLIENT -nm -q " query_id=$(random_str 10) $CLICKHOUSE_CLIENT --query_id "$query_id" -q "backup table data to S3(s3_conn, 'backups/$CLICKHOUSE_DATABASE/data/backup3') SETTINGS allow_s3_native_copy=0" --max_backup_bandwidth=1M > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " SYSTEM FLUSH LOGS; SELECT 'no_native_copy', diff --git a/tests/queries/0_stateless/02867_storage_set_tsan.sh b/tests/queries/0_stateless/02867_storage_set_tsan.sh index 81ae5f0bda8..f6b684c0a55 100755 --- a/tests/queries/0_stateless/02867_storage_set_tsan.sh +++ b/tests/queries/0_stateless/02867_storage_set_tsan.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -mn -q """ +$CLICKHOUSE_CLIENT -m -q """ DROP TABLE IF EXISTS t1_02867; CREATE TABLE t1_02867 (x UInt64) ENGINE=Set(); """ @@ -39,4 +39,4 @@ repeat_truncate_insert & sleep 10 -$CLICKHOUSE_CLIENT -mn -q "DROP TABLE IF EXISTS t1_02867;" +$CLICKHOUSE_CLIENT -m -q "DROP TABLE IF EXISTS t1_02867;" diff --git a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh index a0fdb5276e0..a091645d1e3 100755 --- a/tests/queries/0_stateless/02900_union_schema_inference_mode.sh +++ b/tests/queries/0_stateless/02900_union_schema_inference_mode.sh @@ -10,14 +10,14 @@ echo '{"a" : 1, "obj" : {"f1" : 1, "f2" : "2020-01-01"}}' > $CLICKHOUSE_TEST_UNI echo '{"b" : 2, "obj" : {"f3" : 2, "f2" : "Some string"}}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data2.jsonl echo '{"c" : "hello"}' > $CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); @@ -25,14 +25,14 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3}.jsonl'); cd $CLICKHOUSE_TEST_UNIQUE_NAME/ && tar -cf archive.tar data1.jsonl data2.jsonl data3.jsonl && cd .. -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl') order by tuple(*) format JSONEachRow; select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache order by file; " -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data3.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); @@ -41,7 +41,7 @@ desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/archive.tar :: data{1,2,3}.jsonl'); echo 'Error' > $CLICKHOUSE_TEST_UNIQUE_NAME/data4.jsonl $CLICKHOUSE_LOCAL -q "desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl') settings schema_inference_mode='union'" 2>&1 | grep -c -F "CANNOT_EXTRACT_TABLE_STRUCTURE" -$CLICKHOUSE_LOCAL -nm -q " +$CLICKHOUSE_LOCAL -m -q " set schema_inference_mode = 'union'; desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{2,3}.jsonl'); desc file('$CLICKHOUSE_TEST_UNIQUE_NAME/data{1,2,3,4}.jsonl'); diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 81ba59fc591..91eeb22c19e 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -67,7 +67,7 @@ curl "$CLICKHOUSE_URL" --silent --fail --show-error --data "SELECT sum(is_leader wait; -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " SYSTEM FLUSH LOGS; -- Check that number of ZK request is less then a half of (total replicas * concurrency) diff --git a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh index d1cbc54d294..2eccade5c81 100755 --- a/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh +++ b/tests/queries/0_stateless/02922_deduplication_with_zero_copy.sh @@ -8,7 +8,7 @@ CURDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists r1; drop table if exists r2; @@ -64,7 +64,7 @@ function insert_duplicates() { wait - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " system sync replica r1; system sync replica r2; " @@ -84,7 +84,7 @@ function loop() do while ! insert_duplicates do - $CLICKHOUSE_CLIENT -nm -q " + $CLICKHOUSE_CLIENT -m -q " truncate table r1; truncate table r2; system sync replica r1; @@ -137,8 +137,8 @@ function list_keeper_nodes() { list_keeper_nodes "${table_shared_id}" -$CLICKHOUSE_CLIENT -nm -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & -$CLICKHOUSE_CLIENT -nm -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -m -q "drop table r1;" --allow_repeated_settings --send_logs_level="error" & +$CLICKHOUSE_CLIENT -m -q "drop table r2;" --allow_repeated_settings --send_logs_level="error" & wait list_keeper_nodes "${table_shared_id}" diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh index 2b92a113e91..057f76e63d0 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_1.sh @@ -10,11 +10,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Basic refreshing. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view a refresh after 2 second engine Memory @@ -23,41 +23,41 @@ $CLICKHOUSE_CLIENT -nq " select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes; show create a;" # Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] do sleep 0.5 done -start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" +start_time="`$CLICKHOUSE_CLIENT -q "select reinterpret(now64(), 'Int64')"`" # Check table contents. -$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" +$CLICKHOUSE_CLIENT -q "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a" # Wait for table contents to change. -res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" +res1="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values'`" while : do - res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + res2="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" [ "$res2" == "$res1" ] || break sleep 0.5 done # Wait for another change. while : do - res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" + res3="`$CLICKHOUSE_CLIENT -q 'select * from a order by x format Values -- $LINENO'`" [ "$res3" == "$res2" ] || break sleep 0.5 done # Check that the two changes were at least 1 second apart, in particular that we're not refreshing # like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer # to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000); select '<4: next refresh in>', next_refresh_time-last_refresh_time from refreshes;" # Create a source table from which views will read. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table src (x Int8) engine Memory as select 1;" # Switch to fake clock, change refresh schedule, change query. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view a set fake time '2050-01-01 00:00:01'; system wait view a; system refresh view a; @@ -68,19 +68,19 @@ $CLICKHOUSE_CLIENT -nq " select '<4.5: altered>', status, last_refresh_result, next_refresh_time from refreshes; show create a;" # Advance time to trigger the refresh. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<5: no refresh>', count() from a; system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<6: refreshed>', * from a; select '<7: refreshed>', status, last_refresh_result, next_refresh_time from refreshes;" # Create a dependent view, refresh it once. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a; show create b; system test view b set fake time '2052-11-11 11:11:11'; @@ -88,89 +88,89 @@ $CLICKHOUSE_CLIENT -nq " system wait view b; select '<7.5: created dependent>', last_refresh_time from refreshes where view = 'b';" # Next refresh shouldn't start until the dependency refreshes. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<8: refreshed>', * from b; select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes; system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] do sleep 0.5 done # Drop the source table, check that refresh fails and doesn't leave a temp table behind. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<9.2: dropping>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase(); drop table src; system refresh view a;" -$CLICKHOUSE_CLIENT -nq "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q "system wait view a;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -q " select '<9.4: dropped>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase();" # Create the source table again, check that refresh succeeds (in particular that tables are looked # up by name rather than uuid). -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<10: creating>', view, status, remaining_dependencies, next_refresh_time from refreshes; create table src (x Int16) engine Memory as select 2; system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Both tables should've refreshed. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<11: chain-refreshed a>', * from a; select '<12: chain-refreshed b>', * from b; select '<13: chain-refreshed>', view, status, remaining_dependencies, last_refresh_result, last_refresh_time, next_refresh_time, exception == '' from refreshes;" # Make the dependent table run ahead by one refresh cycle, make sure it waits for the dependency to # catch up to the same cycle. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2059-01-01 00:00:00'; system refresh view b;" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2061-01-01 00:00:00'; system test view a set fake time '2057-01-01 00:00:00';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<14: waiting for next cycle>', view, status, remaining_dependencies, next_refresh_time from refreshes; truncate src; insert into src values (3); system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<15: chain-refreshed a>', * from a; select '<16: chain-refreshed b>', * from b; select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;" # Get to WaitingForDependencies state and remove the depencency. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<18: removed dependency>', view, status, remaining_dependencies, last_refresh_time,next_refresh_time, refresh_count from refreshes where view = 'b'; show create b;" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table src; drop table a; drop table b; diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh index 50a905576d5..2d00d61f253 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views_2.sh @@ -12,29 +12,29 @@ CLICKHOUSE_LOG_COMMENT= CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`" CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --allow_materialized_view_with_bad_select=0 --session_timezone Etc/UTC"`" -$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" +$CLICKHOUSE_CLIENT -q "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view" # Select from a table that doesn't exist, get an exception. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table src (x Int8) engine Memory as select 1; create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src; drop table src;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Error' ] do sleep 0.5 done # Check exception, create src, expect successful refresh. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' ? '1' : exception from refreshes where view = 'c'; create table src (x Int64) engine Memory as select 1; system refresh view c;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Rename table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<20: unexception>', * from c; rename table c to d; select '<21: rename>', * from d; @@ -42,130 +42,130 @@ $CLICKHOUSE_CLIENT -nq " # Do various things during a refresh. # First make a nonempty view. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table d; truncate src; insert into src values (1); create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x empty as select x + sleepEachRow(1) as x from src settings max_block_size = 1;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done # Stop refreshes. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<23: simple refresh>', * from e; system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] do sleep 0.5 done # Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure # we wait for a slow refresh, not a previous fast one.) -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " insert into src select * from numbers(1000) settings max_block_size=1; system start view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Rename. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " rename table e to f; select '<24: rename during refresh>', * from f; select '<25: rename during refresh>', view, status from refreshes where view = 'f'; alter table f modify refresh after 10 year;" # Cancel. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.5 done # Check that another refresh doesn't immediately start after the cancelled one. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<27: cancelled>', view, status, last_refresh_result from refreshes where view = 'f'; system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -q "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ] do sleep 0.5 done # Drop. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table f; select '<28: drop during refresh>', view, status from refreshes; select '<28: drop during refresh>', countIf(name like '%tmp%'), countIf(name like '%.inner%') from system.tables where database = currentDatabase()" # Try OFFSET and RANDOMIZE FOR. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42 as x; show create g; system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] +while [ "`$CLICKHOUSE_CLIENT -q "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " with '2050-02-10 04:00:00'::DateTime as expected select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;" # Send data 'TO' an existing table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table g; create table dest (x Int64) engine MergeTree order by x; truncate src; insert into src values (1); create materialized view h refresh every 1 second to dest empty as select x*10 as x from src; show create h;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<30: to existing table>', * from dest; insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] +while [ "`$CLICKHOUSE_CLIENT -q "select count() from dest -- $LINENO" | xargs`" != '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<31: to existing table>', * from dest; drop table dest; drop table h;" # Retries. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view h2 refresh after 1 year settings refresh_retries = 10 (x Int64) engine Memory as select x*10 + throwIf(x % 2 == 0) as x from src;" -$CLICKHOUSE_CLIENT -nq "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q "system wait view h2;" 2>/dev/null && echo "SYSTEM WAIT VIEW failed to fail at $LINENO" +$CLICKHOUSE_CLIENT -q " select '<31.5: will retry>', last_refresh_result, retry > 0 from refreshes; create table src2 (x Int8) engine Memory; insert into src2 values (1); exchange tables src and src2; drop table src2;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] +while [ "`$CLICKHOUSE_CLIENT -q "select last_refresh_result, retry from refreshes -- $LINENO" | xargs`" != 'Finished 0' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<31.6: did retry>', x from h2; drop table h2" # EMPTY -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2); create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] +while [ "`$CLICKHOUSE_CLIENT -q "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ] do sleep 0.5 done -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " select '<32: empty>', view, status, last_refresh_result, retry from refreshes order by view; drop table i; drop table j;" # APPEND -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view k refresh every 10 year append (x Int64) engine Memory empty as select x*10 as x from src; select '<33: append>', * from k; system refresh view k; @@ -177,7 +177,7 @@ $CLICKHOUSE_CLIENT -nq " system wait view k; select '<35: append>', * from k order by x;" # ALTER to non-APPEND -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " alter table k modify refresh every 10 year; system wait view k; system refresh view k; @@ -187,7 +187,7 @@ $CLICKHOUSE_CLIENT -nq " truncate table src;" # APPEND + TO + regular materialized view reading from it. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create table mid (x Int64) engine MergeTree order by x; create materialized view l refresh every 10 year append to mid empty as select x*10 as x from src; create materialized view m (x Int64) engine Memory as select x*10 as x from mid; @@ -204,19 +204,19 @@ $CLICKHOUSE_CLIENT -nq " drop table mid;" # Failing to create inner table. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view n refresh every 1 second (x Int64) engine MergeTree as select 1 as x from numbers(2);" 2>/dev/null || echo "creating MergeTree without ORDER BY failed, as expected" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view n refresh every 1 second (x Int64) engine MergeTree order by x as select 1 as x from numbers(2); drop table n;" # Reading from table that doesn't exist yet. -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nonexist; -- { serverError UNKNOWN_TABLE } create materialized view o (x Int64) engine Memory as select x from nope.nonexist; -- { serverError UNKNOWN_DATABASE } create materialized view o refresh every 1 second (x Int64) engine Memory as select x from nope.nonexist settings allow_materialized_view_with_bad_select = 1; drop table o;" -$CLICKHOUSE_CLIENT -nq " +$CLICKHOUSE_CLIENT -q " drop table refreshes;" diff --git a/tests/queries/0_stateless/02941_variant_type_1.sh b/tests/queries/0_stateless/02941_variant_type_1.sh index c12bece6d54..ef1d1deff4e 100755 --- a/tests/queries/0_stateless/02941_variant_type_1.sh +++ b/tests/queries/0_stateless/02941_variant_type_1.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test1_insert() { echo "test1 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number from numbers(3); insert into test select number + 6, ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(3); @@ -21,7 +21,7 @@ insert into test select number + 15, range(number + 1)::Array(UInt64) from numbe function test1_select() { echo "test1 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -36,7 +36,7 @@ select v.\`Array(UInt64)\`.size0 from test order by id;" function test2_insert() { echo "test2 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(3); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(3); insert into test select number + 3, number % 2 ? NULL : number from numbers(3); insert into test select number + 6, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(3); insert into test select number + 9, number % 2 ? CAST(NULL, 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') : CAST(('lc_str_' || toString(number))::LowCardinality(String), 'Variant(String, UInt64, LowCardinality(String), Tuple(a UInt32, b UInt32), Array(UInt64))') from numbers(3); @@ -47,7 +47,7 @@ insert into test select number + 15, number % 2 ? CAST(NULL, 'Variant(String, UI function test2_select() { echo "test2 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; @@ -68,7 +68,7 @@ function test3_insert() function test3_select() { echo "test3 select" - $CH_CLIENT -nmq "select v from test order by id; + $CH_CLIENT -mq "select v from test order by id; select v.String from test order by id; select v.UInt64 from test order by id; select v.\`LowCardinality(String)\` from test order by id; diff --git a/tests/queries/0_stateless/02941_variant_type_2.sh b/tests/queries/0_stateless/02941_variant_type_2.sh index e93dfac8510..7017edac525 100755 --- a/tests/queries/0_stateless/02941_variant_type_2.sh +++ b/tests/queries/0_stateless/02941_variant_type_2.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test4_insert() { echo "test4 insert" - $CH_CLIENT -nmq "insert into test select number, NULL from numbers(100000); + $CH_CLIENT -mq "insert into test select number, NULL from numbers(100000); insert into test select number + 100000, number from numbers(100000); insert into test select number + 200000, ('str_' || toString(number))::Variant(String) from numbers(100000); insert into test select number + 300000, ('lc_str_' || toString(number))::LowCardinality(String) from numbers(100000); @@ -21,7 +21,7 @@ insert into test select number + 500000, range(number % 20 + 1)::Array(UInt64) f function test4_select { echo "test4 select" - $CH_CLIENT -nmq "select v from test format Null; + $CH_CLIENT -mq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02941_variant_type_3.sh b/tests/queries/0_stateless/02941_variant_type_3.sh index cc0fde5b689..5d923a47e9b 100755 --- a/tests/queries/0_stateless/02941_variant_type_3.sh +++ b/tests/queries/0_stateless/02941_variant_type_3.sh @@ -10,7 +10,7 @@ CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --allow_suspic function test5_insert() { echo "test5 insert" - $CH_CLIENT -nmq " + $CH_CLIENT -mq " insert into test select number, NULL from numbers(200000); insert into test select number + 200000, number % 2 ? NULL : number from numbers(200000); insert into test select number + 400000, number % 2 ? NULL : ('str_' || toString(number))::Variant(String) from numbers(200000); @@ -22,7 +22,7 @@ insert into test select number + 1000000, number % 2 ? CAST(NULL, 'Variant(Strin function test5_select() { echo "test5 select" - $CH_CLIENT -nmq " + $CH_CLIENT -mq " select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; diff --git a/tests/queries/0_stateless/02941_variant_type_4.sh b/tests/queries/0_stateless/02941_variant_type_4.sh index 93a1770d05e..88f6a475d46 100755 --- a/tests/queries/0_stateless/02941_variant_type_4.sh +++ b/tests/queries/0_stateless/02941_variant_type_4.sh @@ -17,7 +17,7 @@ function test6_insert() function test6_select() { echo "test6 select" - $CH_CLIENT -nmq "select v from test format Null; + $CH_CLIENT -mq "select v from test format Null; select count() from test where isNotNull(v); select v.String from test format Null; select count() from test where isNotNull(v.String); diff --git a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh index cb099bb59ae..f9e84dc2b50 100755 --- a/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh +++ b/tests/queries/0_stateless/02944_dynamically_change_filesystem_cache_size.sh @@ -10,7 +10,7 @@ disk_name="s3_cache_02944" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(100); @@ -33,7 +33,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -47,7 +47,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -63,7 +63,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" @@ -77,7 +77,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" $CLICKHOUSE_CLIENT --query "DESCRIBE FILESYSTEM CACHE '${disk_name}'" diff --git a/tests/queries/0_stateless/02960_polygon_bound_bug.sh b/tests/queries/0_stateless/02960_polygon_bound_bug.sh index 0c3db01a77c..1e9be36da55 100755 --- a/tests/queries/0_stateless/02960_polygon_bound_bug.sh +++ b/tests/queries/0_stateless/02960_polygon_bound_bug.sh @@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_LOCAL -nm -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; +$CLICKHOUSE_LOCAL -m -q "CREATE TABLE test_table (geom MultiPolygon) engine=MergeTree ORDER BY geom; INSERT INTO test_table SELECT * FROM file('$CURDIR/data_parquet/02960_polygon_bound_bug.parquet', Parquet); CREATE DICTIONARY test_dict (geom MultiPolygon) PRIMARY KEY geom SOURCE (CLICKHOUSE(TABLE 'test_table')) LIFETIME(MIN 0 MAX 0) LAYOUT(POLYGON(STORE_POLYGON_KEY_COLUMN 1)); SELECT dictHas(test_dict,(174.84729269276494,-36.99524960275426));" diff --git a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh index 145b921a750..1bd86ee0c75 100755 --- a/tests/queries/0_stateless/02961_storage_config_volume_priority.sh +++ b/tests/queries/0_stateless/02961_storage_config_volume_priority.sh @@ -24,7 +24,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must be unique across the policy' @@ -40,7 +40,7 @@ cat $config_path \ > $config_path_tmp mv $config_path_tmp $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='error'; SYSTEM RELOAD CONFIG" 2>&1 | grep -c 'volume_priority values must cover the range from 1 to N (lowest priority specified) without gaps' diff --git a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh index 24fe964b824..3b57941452d 100755 --- a/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh +++ b/tests/queries/0_stateless/02963_remote_read_small_buffer_size_bug.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="02963_remote_read_bug" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a Int32, s String) @@ -22,7 +22,7 @@ OPTIMIZE TABLE test FINAL; query_id=$(random_str 10) -$CLICKHOUSE_CLIENT -nm --query_id "$query_id" --query " +$CLICKHOUSE_CLIENT -m --query_id "$query_id" --query " WITH RANDOM_SET AS ( SELECT rand32() % 10000 FROM numbers(100) ) @@ -37,7 +37,7 @@ SETTINGS merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem = 1, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem = 1; " -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " SYSTEM FLUSH LOGS; -- This threshold was determined experimentally - before the fix this ratio had values around 50K diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh index 1089eb4051f..b4271c3d29b 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh index 7a0e2d9bfdb..ed13bf3321b 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh index a595e363ef4..4be4ab81c68 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists num_1; drop table if exists num_2; diff --git a/tests/queries/0_stateless/02969_auto_format_detection.sh b/tests/queries/0_stateless/02969_auto_format_detection.sh index 88d6575e499..ab21dbb3df8 100755 --- a/tests/queries/0_stateless/02969_auto_format_detection.sh +++ b/tests/queries/0_stateless/02969_auto_format_detection.sh @@ -24,12 +24,12 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE', auto, 'a UInt64, b String, c Array(UInt64), d Tuple(a UInt64, b String)')" -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE'); desc file('$DATA_FILE'); " -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE', JSONEachRow); desc file('$DATA_FILE'); " @@ -39,7 +39,7 @@ $CLICKHOUSE_LOCAL -q "select * from generateRandom('a UInt64, b String, c Array( $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}')" $CLICKHOUSE_LOCAL -q "desc file('$DATA_FILE.{1,2}') settings schema_inference_mode='union'" 2>&1 | grep -c "CANNOT_DETECT_FORMAT" -$CLICKHOUSE_LOCAL -nmq " +$CLICKHOUSE_LOCAL -mq " desc file('$DATA_FILE.2'); desc file('$DATA_FILE.{1,2}'); " diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh index d543f7195a9..a46f0018ad3 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_MergeTree.sh @@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data; create table data (key Int) engine=MergeTree() order by key; insert into data values (1); @@ -29,7 +29,7 @@ $CLICKHOUSE_CLIENT -nm -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table data; attach table data (key Int) engine=MergeTree() order by key settings diff --git a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh index eec05c81344..522fdd6096a 100755 --- a/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh +++ b/tests/queries/0_stateless/02980_s3_plain_DROP_TABLE_ReplicatedMergeTree.sh @@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -q "create database $n CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT/--database=$CLICKHOUSE_DATABASE/--database=$new_database} CLICKHOUSE_DATABASE="$new_database" -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table if exists data_read; drop table if exists data_write; @@ -33,7 +33,7 @@ $CLICKHOUSE_CLIENT -nm -q " # suppress output $CLICKHOUSE_CLIENT -q "backup table data_read to S3('http://localhost:11111/test/s3_plain/backups/$CLICKHOUSE_DATABASE', 'test', 'testtest')" > /dev/null -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " drop table data_read; attach table data_read (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', 'read') order by key settings @@ -57,7 +57,7 @@ echo "Files before DETACH TABLE" # sed to match any part, since in case of fault injection part name may not be all_0_0_0 but all_1_1_0 clickhouse-disks -C "$config" --disk s3_plain_disk --query "list --recursive $path" | tail -n+2 | sed 's/all_[^_]*_[^_]*_0/all_X_X_X/g' -$CLICKHOUSE_CLIENT -nm -q " +$CLICKHOUSE_CLIENT -m -q " detach table data_read; detach table data_write; " diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh index ef4866cb69e..fca8b7a5e97 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh index dd4b1f7cec0..08d5ae3a6b2 100755 --- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh index 33386c76edb..678b1db4d68 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh index b66ef83abf2..606fda6d4fd 100755 --- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh index f9e1838f491..b571631dc8f 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh index 698e70d4064..e4358011e50 100755 --- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh +++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh @@ -36,7 +36,7 @@ for insert_method in "InsertSelect" "InsertValues"; do fi echo "$THIS_RUN" - $CLICKHOUSE_CLIENT --max_insert_block_size 1 -nmq " + $CLICKHOUSE_CLIENT --max_insert_block_size 1 -mq " $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \ --insert-method $insert_method \ --table-engine $ENGINE \ diff --git a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh index 8eea7940774..c13674b9bd1 100755 --- a/tests/queries/0_stateless/03008_s3_plain_rewritable.sh +++ b/tests/queries/0_stateless/03008_s3_plain_rewritable.sh @@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query "drop table if exists test_s3_mt" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " create table test_s3_mt (a Int32, b Int64, c Int64) engine = MergeTree() partition by intDiv(a, 1000) order by tuple(a, b) settings disk = disk( name = 03008_s3_plain_rewritable, @@ -19,7 +19,7 @@ settings disk = disk( secret_access_key = clickhouse); " -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " insert into test_s3_mt (*) values (1, 2, 0), (2, 2, 2), (3, 1, 9), (4, 7, 7), (5, 10, 2), (6, 12, 5); insert into test_s3_mt (*) select number, number, number from numbers_mt(10000); select count(*) from test_s3_mt; @@ -31,13 +31,13 @@ ${CLICKHOUSE_CLIENT} --query "optimize table test_s3_mt final" ${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt add projection test_s3_mt_projection (select * order by b)" 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " alter table test_s3_mt update c = 0 where a % 2 = 1; alter table test_s3_mt add column d Int64 after c; alter table test_s3_mt drop column c; " 2>&1 | grep -Fq "SUPPORT_IS_DISABLED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " detach table test_s3_mt; attach table test_s3_mt; " diff --git a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh index cba5317fcfa..278ad6f0654 100755 --- a/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh +++ b/tests/queries/0_stateless/03032_dynamically_resize_filesystem_cache_2.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) disk_name="s3_cache" -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " DROP TABLE IF EXISTS test; CREATE TABLE test (a String) engine=MergeTree() ORDER BY tuple() SETTINGS disk = '$disk_name'; INSERT INTO test SELECT randomString(1000); @@ -26,7 +26,7 @@ sed -i "s|$prev_max_size<\/max_size>|$new_max_size<\/max_siz # echo $prev_max_size # echo $new_max_size -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" @@ -36,7 +36,7 @@ $CLICKHOUSE_CLIENT --query "SELECT current_size <= max_size FROM system.filesyst sed -i "s|$new_max_size<\/max_size>|$prev_max_size<\/max_size>|" $config_path -$CLICKHOUSE_CLIENT -nm --query " +$CLICKHOUSE_CLIENT -m --query " set send_logs_level='fatal'; SYSTEM RELOAD CONFIG" diff --git a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh index 9ea86105a3a..8183b636549 100755 --- a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from (select sumMerge(sum) as sum from test group by id, _part) group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh index 9a2a6dd957c..ddafa9db210 100755 --- a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh @@ -16,7 +16,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh index 0199035a3df..b1046ca08ba 100755 --- a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh index e2ea5bc3466..f3693a2e7b1 100755 --- a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "select count(), sum from test group by sum order by sum, count()" $CH_CLIENT -q "drop table test" diff --git a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh index 43607cf95a8..bba92a2ee06 100755 --- a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh +++ b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh @@ -17,7 +17,7 @@ function test() $CH_CLIENT -q "insert into test select number, -1, number >= 75000 ? 2 : 1, 'str_' || toString(number) from numbers(50000, 100000)" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" - $CH_CLIENT -nm -q "system start merges test; optimize table test final" + $CH_CLIENT -m -q "system start merges test; optimize table test final" $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)" $CH_CLIENT -q "drop table test" } diff --git a/tests/queries/0_stateless/03151_unload_index_race.sh b/tests/queries/0_stateless/03151_unload_index_race.sh index 7e9dfa7cddc..4cdf06abae2 100755 --- a/tests/queries/0_stateless/03151_unload_index_race.sh +++ b/tests/queries/0_stateless/03151_unload_index_race.sh @@ -42,8 +42,8 @@ function thread_alter_settings() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - $CLICKHOUSE_CLIENT -n --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" - $CLICKHOUSE_CLIENT -n --query "SYSTEM UNLOAD PRIMARY KEY t" + $CLICKHOUSE_CLIENT --query "ALTER TABLE t MODIFY SETTING primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns=0.$RANDOM" + $CLICKHOUSE_CLIENT --query "SYSTEM UNLOAD PRIMARY KEY t" sleep 0.0$RANDOM done } @@ -52,7 +52,7 @@ function thread_query_table() { local TIMELIMIT=$((SECONDS+$1)) while [ $SECONDS -lt "$TIMELIMIT" ]; do - COUNT=$($CLICKHOUSE_CLIENT -n --query "SELECT count() FROM t where not ignore(*);") + COUNT=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM t where not ignore(*);") if [ "$COUNT" -ne "2000" ]; then echo "$COUNT" fi diff --git a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh index 22a2fd82c64..d1afafa77a5 100755 --- a/tests/queries/0_stateless/03172_error_log_table_not_empty.sh +++ b/tests/queries/0_stateless/03172_error_log_table_not_empty.sh @@ -15,7 +15,7 @@ errors_222=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHER errors_333=$($CLICKHOUSE_CLIENT -q "SELECT sum(value) FROM system.error_log WHERE code = 333") # Throw three random errors: 111, 222 and 333 and wait for more than collect_interval_milliseconds to ensure system.error_log is flushed -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -24,14 +24,14 @@ SYSTEM FLUSH LOGS; " # Check that the three random errors are propagated -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT sum(value) > $errors_111 FROM system.error_log WHERE code = 111; SELECT sum(value) > $errors_222 FROM system.error_log WHERE code = 222; SELECT sum(value) > $errors_333 FROM system.error_log WHERE code = 333; " # Ensure that if we throw them again, they're still propagated -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT throwIf(true, 'error_log', toInt16(111)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 111 } SELECT throwIf(true, 'error_log', toInt16(222)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 222 } SELECT throwIf(true, 'error_log', toInt16(333)) SETTINGS allow_custom_error_code_in_throwif=1; -- { serverError 333 } @@ -39,7 +39,7 @@ SELECT sleep(2) format NULL; SYSTEM FLUSH LOGS; " -$CLICKHOUSE_CLIENT -mn -q " +$CLICKHOUSE_CLIENT -m -q " SELECT sum(value) > $(($errors_111+1)) FROM system.error_log WHERE code = 111; SELECT sum(value) > $(($errors_222+1)) FROM system.error_log WHERE code = 222; SELECT sum(value) > $(($errors_333+1)) FROM system.error_log WHERE code = 333; diff --git a/tests/queries/0_stateless/03203_hive_style_partitioning.sh b/tests/queries/0_stateless/03203_hive_style_partitioning.sh index 60e8a6e9faa..fdd0fcd8ec5 100755 --- a/tests/queries/0_stateless/03203_hive_style_partitioning.sh +++ b/tests/queries/0_stateless/03203_hive_style_partitioning.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE FILE HIVE PARTITIONING'" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -22,20 +22,20 @@ SELECT toTypeName(array), toTypeName(float) FROM file('$CURDIR/data_hive/partiti SELECT count(*) FROM file('$CURDIR/data_hive/partitioning/number=42/date=2020-01-01/sample.parquet') WHERE number = 42; """ -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT identifier FROM file('$CURDIR/data_hive/partitioning/identifier=*/email.csv') LIMIT 2; SELECT a FROM file('$CURDIR/data_hive/partitioning/a=b/a=b/sample.parquet') LIMIT 1; """ -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM file('$CURDIR/data_hive/partitioning/column0=Elizabeth/column0=Elizabeth1/sample.parquet') LIMIT 10; """ 2>&1 | grep -c "INCORRECT_DATA" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 0; SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10; @@ -45,14 +45,14 @@ SELECT *, non_existing_column FROM file('$CURDIR/data_hive/partitioning/non_exis $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE URL PARTITIONING'" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; SELECT *, non_existing_column FROM url('http://localhost:11111/test/hive_partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10;""" -$CLICKHOUSE_LOCAL -n -q """ +$CLICKHOUSE_LOCAL -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -62,7 +62,7 @@ SELECT *, _column0 FROM url('http://localhost:11111/test/hive_partitioning/colum $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3 PARTITIONING'" -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -71,7 +71,7 @@ SELECT *, non_existing_column FROM s3('http://localhost:11111/test/hive_partitio SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=*/sample.parquet') WHERE column0 = 'Elizabeth' LIMIT 10; """ -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 0; SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; @@ -79,7 +79,7 @@ SELECT *, _column0 FROM s3('http://localhost:11111/test/hive_partitioning/column $CLICKHOUSE_LOCAL -q "SELECT 'TESTING THE S3CLUSTER PARTITIONING'" -$CLICKHOUSE_CLIENT -n -q """ +$CLICKHOUSE_CLIENT -q """ set use_hive_partitioning = 1; SELECT *, column0 FROM s3Cluster(test_cluster_one_shard_three_replicas_localhost, 'http://localhost:11111/test/hive_partitioning/column0=Elizabeth/sample.parquet') LIMIT 10; diff --git a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh index e0c8f08e695..6bfe82a91a6 100755 --- a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh +++ b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # In this test we restore from "/tests/queries/0_stateless/backups/mt_250_parts.zip" backup_name="$($CURDIR/helpers/install_predefined_backup.sh mt_250_parts.zip)" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " DROP TABLE IF EXISTS manyparts; CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merge_tree_clear_old_temporary_directories_interval_seconds=1, temporary_directories_lifetime=1; " @@ -16,7 +16,7 @@ CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merg # RESTORE must protect its temporary directories from removing. ${CLICKHOUSE_CLIENT} --query "RESTORE TABLE default.mt_250_parts AS manyparts FROM Disk('backups', '${backup_name}') SETTINGS allow_different_table_def=true" | grep -o "RESTORED" -${CLICKHOUSE_CLIENT} -nm --query " +${CLICKHOUSE_CLIENT} -m --query " SELECT count(), sum(x) FROM manyparts; DROP TABLE manyparts; " From e5ae4c8e8bc0b210d75a23494d74e879adb83f02 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:19:20 +0200 Subject: [PATCH 145/148] Update docs of JSONAsObject format with new JSON type --- docs/en/interfaces/formats.md | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 4cf8b8bd1c5..2dd7c1dbfb6 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -826,17 +826,17 @@ Result: ## JSONAsObject {#jsonasobject} -In this format, a single JSON object is interpreted as a single [Object('json')](/docs/en/sql-reference/data-types/json.md) value. If the input has several JSON objects (comma separated), they are interpreted as separate rows. If the input data is enclosed in square brackets, it is interpreted as an array of JSONs. +In this format, a single JSON object is interpreted as a single [JSON](/docs/en/sql-reference/data-types/newjson.md) value. If the input has several JSON objects (comma separated), they are interpreted as separate rows. If the input data is enclosed in square brackets, it is interpreted as an array of JSONs. -This format can only be parsed for a table with a single field of type [Object('json')](/docs/en/sql-reference/data-types/json.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized). +This format can only be parsed for a table with a single field of type [JSON](/docs/en/sql-reference/data-types/newjson.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized). **Examples** Query: ``` sql -SET allow_experimental_object_type = 1; -CREATE TABLE json_as_object (json Object('json')) ENGINE = Memory; +SET allow_experimental_json_type = 1; +CREATE TABLE json_as_object (json JSON) ENGINE = Memory; INSERT INTO json_as_object (json) FORMAT JSONAsObject {"foo":{"bar":{"x":"y"},"baz":1}},{},{"any json stucture":1} SELECT * FROM json_as_object FORMAT JSONEachRow; ``` @@ -844,9 +844,9 @@ SELECT * FROM json_as_object FORMAT JSONEachRow; Result: ``` response -{"json":{"any json stucture":0,"foo":{"bar":{"x":"y"},"baz":1}}} -{"json":{"any json stucture":0,"foo":{"bar":{"x":""},"baz":0}}} -{"json":{"any json stucture":1,"foo":{"bar":{"x":""},"baz":0}}} +{"json":{"foo":{"bar":{"x":"y"},"baz":"1"}}} +{"json":{}} +{"json":{"any json stucture":"1"}} ``` **An array of JSON objects** @@ -854,35 +854,34 @@ Result: Query: ``` sql -SET allow_experimental_object_type = 1; -CREATE TABLE json_square_brackets (field Object('json')) ENGINE = Memory; +SET allow_experimental_json_type = 1; +CREATE TABLE json_square_brackets (field JSON) ENGINE = Memory; INSERT INTO json_square_brackets FORMAT JSONAsObject [{"id": 1, "name": "name1"}, {"id": 2, "name": "name2"}]; - SELECT * FROM json_square_brackets FORMAT JSONEachRow; ``` Result: ```response -{"field":{"id":1,"name":"name1"}} -{"field":{"id":2,"name":"name2"}} +{"field":{"id":"1","name":"name1"}} +{"field":{"id":"2","name":"name2"}} ``` **Columns with default values** ```sql -SET allow_experimental_object_type = 1; -CREATE TABLE json_as_object (json Object('json'), time DateTime MATERIALIZED now()) ENGINE = Memory; +SET allow_experimental_json_type = 1; +CREATE TABLE json_as_object (json JSON, time DateTime MATERIALIZED now()) ENGINE = Memory; INSERT INTO json_as_object (json) FORMAT JSONAsObject {"foo":{"bar":{"x":"y"},"baz":1}}; INSERT INTO json_as_object (json) FORMAT JSONAsObject {}; INSERT INTO json_as_object (json) FORMAT JSONAsObject {"any json stucture":1} -SELECT * FROM json_as_object FORMAT JSONEachRow +SELECT time, json FROM json_as_object FORMAT JSONEachRow ``` ```resonse -{"json":{"any json stucture":0,"foo":{"bar":{"x":"y"},"baz":1}},"time":"2024-07-25 17:02:45"} -{"json":{"any json stucture":0,"foo":{"bar":{"x":""},"baz":0}},"time":"2024-07-25 17:02:47"} -{"json":{"any json stucture":1,"foo":{"bar":{"x":""},"baz":0}},"time":"2024-07-25 17:02:50"} +{"time":"2024-09-16 12:18:10","json":{}} +{"time":"2024-09-16 12:18:13","json":{"any json stucture":"1"}} +{"time":"2024-09-16 12:18:08","json":{"foo":{"bar":{"x":"y"},"baz":"1"}}} ``` ## JSONCompact {#jsoncompact} From 23e0e92f5f719a981900edc56aad1a8425ecddd1 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 16 Sep 2024 15:07:40 +0200 Subject: [PATCH 146/148] Fix --- .../queries/0_stateless/02768_into_outfile_extensions_format.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh b/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh index 756488076f9..a33baaf995c 100755 --- a/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh +++ b/tests/queries/0_stateless/02768_into_outfile_extensions_format.sh @@ -9,4 +9,4 @@ select * from numbers(1) into outfile '/dev/null'; select * from numbers(1) into outfile '/dev/null' and stdout; select * from numbers(1) into outfile '/dev/null' append; select * from numbers(1) into outfile '/dev/null' append and stdout; -" | clickhouse-format -n +" | ${CLICKHOUSE_FORMAT} -n From 6a26c5cf8ea3a1a9930e7b2226a3395c45d4c54d Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 16 Sep 2024 20:54:02 +0200 Subject: [PATCH 147/148] Fix --- src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp | 6 +++--- src/Server/StaticRequestHandler.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 946eaf8aea4..47af568838a 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -17,9 +17,7 @@ void WriteBufferFromHTTPServerResponse::startSendHeaders() { headers_started_sending = true; - if (response.getChunkedTransferEncoding()) - setChunked(); - else if (response.getContentLength() == Poco::Net::HTTPMessage::UNKNOWN_CONTENT_LENGTH) + if (!response.getChunkedTransferEncoding() && response.getContentLength() == Poco::Net::HTTPMessage::UNKNOWN_CONTENT_LENGTH) { /// In case there is no Content-Length we cannot use keep-alive, /// since there is no way to know when the server send all the @@ -134,6 +132,8 @@ WriteBufferFromHTTPServerResponse::WriteBufferFromHTTPServerResponse( , response(response_) , is_http_method_head(is_http_method_head_) { + if (response.getChunkedTransferEncoding()) + setChunked(); } diff --git a/src/Server/StaticRequestHandler.cpp b/src/Server/StaticRequestHandler.cpp index d8c0765bca4..f0633eb1c70 100644 --- a/src/Server/StaticRequestHandler.cpp +++ b/src/Server/StaticRequestHandler.cpp @@ -90,15 +90,15 @@ static inline void trySendExceptionToClient( void StaticRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & /*write_event*/) { + applyHTTPResponseHeaders(response, http_response_headers_override); + + if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) + response.setChunkedTransferEncoding(true); + auto out = responseWriteBuffer(request, response); try { - applyHTTPResponseHeaders(response, http_response_headers_override); - - if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) - response.setChunkedTransferEncoding(true); - /// Workaround. Poco does not detect 411 Length Required case. if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && !request.hasContentLength()) throw Exception(ErrorCodes::HTTP_LENGTH_REQUIRED, From 8c7c37de1d47c6a63cad0e3e27080a3c116d21c3 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 16 Sep 2024 23:41:51 +0200 Subject: [PATCH 148/148] temp-commit --- tests/queries/0_stateless/02447_drop_database_replica.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02447_drop_database_replica.sh b/tests/queries/0_stateless/02447_drop_database_replica.sh index abe99398a56..558a0c2693c 100755 --- a/tests/queries/0_stateless/02447_drop_database_replica.sh +++ b/tests/queries/0_stateless/02447_drop_database_replica.sh @@ -46,9 +46,13 @@ timeout 60s $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=1000 --distributed # And that it still throws TIMEOUT_EXCEEDED for active replicas echo 'timeout on active' db9="${db}_9" -$CLICKHOUSE_CLIENT -q "create database $db9 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's9', 'r9')" +REPLICA_UUID=$($CLICKHOUSE_CLIENT -q "select serverUUID()") +if ! [[ $REPLICA_UUID =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]]; then + echo "Weird UUID ${REPLICA_UUID}" +fi +$CLICKHOUSE_CLIENT -q "create database $db9 engine=Replicated('/test/${CLICKHOUSE_DATABASE}/rdb', 's9', 'r9')" $CLICKHOUSE_CLIENT -q "detach database $db9" -$CLICKHOUSE_CLIENT -q "insert into system.zookeeper(name, path, value) values ('active', '/test/$CLICKHOUSE_DATABASE/rdb/replicas/s9|r9', '$($CLICKHOUSE_CLIENT -q "select serverUUID()")')" +$CLICKHOUSE_CLIENT -q "insert into system.zookeeper(name, path, value) values ('active', '/test/${CLICKHOUSE_DATABASE}/rdb/replicas/s9|r9', '${REPLICA_UUID}')" $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=none_only_active -q "create table $db.t22 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED" $CLICKHOUSE_CLIENT --distributed_ddl_task_timeout=5 --distributed_ddl_output_mode=throw_only_active -q "create table $db.t33 (n int) engine=Log" 2>&1| grep -Fac "TIMEOUT_EXCEEDED"